diff --git a/.github/instructions/agent-skills.instructions.md b/.github/instructions/agent-skills.instructions.md index 21ef77485..a68b3aff3 100644 --- a/.github/instructions/agent-skills.instructions.md +++ b/.github/instructions/agent-skills.instructions.md @@ -68,10 +68,9 @@ Before shipping any skill/instruction: ## Mode-specific content -azldev has two modes, and the emitted content follows the one it runs in. A `Catalog` -(`NewCatalog(withoutLockfile)`) resolves the skills, instruction wrappers, and templates; -the package-level `Skills`, `Instructions`, `FindSkill`, `SkillDocument`, and `Files` -helpers are the default (lock-file) mode. +The CLI always emits the default catalog, covering both workflows selected by +`lockless-components.txt`. `NewCatalog(true)` retains an internal all-lockless +variant; the package-level helpers use the default mixed-workflow catalog. - **Shared by default.** Registry entries and templates under `content/` describe the default mode and are used by both, so most edits need nothing extra. @@ -82,11 +81,11 @@ helpers are the default (lock-file) mode. same-named default template for that mode; add one only when the document's content actually differs. -When you add or edit a skill, check whether its content names a command that exists in -only one mode (for example `comp update` versus `comp refresh-upstream-commit`) and, if -so, provide the mode-specific variant. Verify both with -`./out/bin/azldev docs agent show --skill ` and -`./out/bin/azldev --without-lockfile docs agent show --skill `. +When editing a skill, describe which components each command processes: +`comp update` handles locked components; `comp refresh-upstream-commit` handles +listed lockless components. Verify CLI output with +`./out/bin/azldev docs agent show --skill ` and cover catalog variants in +unit tests. There is no global CLI mode selector. ## Config-resolved bindings diff --git a/docs/developer/reference/build.md b/docs/developer/reference/build.md index 965bd46a6..8163b4122 100644 --- a/docs/developer/reference/build.md +++ b/docs/developer/reference/build.md @@ -16,7 +16,7 @@ This repo contains configs for the `golangci-lint` linter. It is installed as pa | Command | Description | When to Use | |---------|-------------|-------------| | `mage -l` | List all available targets | To see all options | -| `mage all` | Build, test, default checks, and scenario pipeline | Before submitting PR | +| `mage all` | Build, then run unit tests, default checks, and scenario tests in parallel; generate docs after they pass | Before submitting PR | | `mage build` | Build Go binaries | After code changes | | `mage install` | Build Go binaries and install | Outer-loop testing | | `mage unit` | Run unit tests | After writing tests | @@ -33,6 +33,13 @@ This repo contains configs for the `golangci-lint` linter. It is installed as pa **Note**: Dependencies run automatically in correct order (i.e. `mage build` and `mage unit` automatically run code generation as needed). +`mage all` shares one code-generation and build pass across its parallel +validation targets. Each validation target must pass before documentation is +regenerated. It still runs the default Go checks, not the optional Python checks; +use `mage check all` to include those. Parallel validation can increase peak +resource use; set `GOMAXPROCS` and `GOFLAGS=-p=` to limit Go concurrency +on memory-constrained hosts. + ## Code Generation The project uses automatic code generation including: diff --git a/docs/user/README.md b/docs/user/README.md index cd9615ddb..1d341cd90 100644 --- a/docs/user/README.md +++ b/docs/user/README.md @@ -5,9 +5,10 @@ - [Create a Project](./how-to/create-project.md) — set up a new Azure Linux project - [Add a Component](./how-to/add-component.md) — import or create a new package - [Build a Component](./how-to/build-component.md) — build RPMs from component definitions +- [Extract a Dist-Git Repository](./how-to/extract-dist-git-repo.md) — export a component's committed files and history - [Build an Image](./how-to/build-image.md) — build and boot Azure Linux images - [Set Up AI Coding Agents](./how-to/set-up-ai-agents.md) — emit agent skill and instruction files -- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — opt in to `--without-lockfile` +- [Preview the Lock-File-Free Mode](./how-to/preview-without-lockfile.md) — select lockless components ## Explanation diff --git a/docs/user/explanation/config-system.md b/docs/user/explanation/config-system.md index ac5059ea9..b87f2de50 100644 --- a/docs/user/explanation/config-system.md +++ b/docs/user/explanation/config-system.md @@ -99,7 +99,7 @@ Component definitions are merged additively. If the same component name (e.g., ` > **Note:** Slice fields (like `overlays`) are **appended**, not replaced, following the same merge behavior used by component configuration inheritance. -> **Preview:** With the global `--without-lockfile` flag, component definitions merge with override semantics instead, and component validation runs only after every included file has been merged, so an individual file may hold a partial component definition. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). +> **Preview:** Components listed in `lockless-components.txt` beside the top-level `azldev.toml` merge with override semantics instead, and their validation runs only after every included file has been merged, so an individual file may hold a partial component definition. Unlisted components retain additive merging and per-file validation. See [Preview the Lock-File-Free Mode](../how-to/preview-without-lockfile.md). ### Component Groups and Images diff --git a/docs/user/how-to/build-component.md b/docs/user/how-to/build-component.md index 8cb5701ef..6bdc4226e 100644 --- a/docs/user/how-to/build-component.md +++ b/docs/user/how-to/build-component.md @@ -26,6 +26,15 @@ Built packages are placed in structured subdirectories under the project's outpu ## Common Options +For components listed in `lockless-components.txt`, render and commit first. +Build extracts the committed component dist-git history using `git-filter-repo`, +fetches configured source files (including download and custom origins), downloads +remaining manifest payloads from lookaside, and builds the SRPM with Mock's `rpmautospec` plugin. +Install `git-filter-repo` and Mock 5.3 or newer. Uncommitted rendered edits are +ignored. Lockless builds reject `--without-git` and `--rpmdev-bumpspec`; +lock-file-based builds are unchanged. See the +[lockless workflow](preview-without-lockfile.md#preparing-querying-and-building-committed-dist-git). + | Flag | Description | |------|-------------| | `--no-check` | Skip the `%check` section (package test suite) | diff --git a/docs/user/how-to/extract-dist-git-repo.md b/docs/user/how-to/extract-dist-git-repo.md new file mode 100644 index 000000000..61a737032 --- /dev/null +++ b/docs/user/how-to/extract-dist-git-repo.md @@ -0,0 +1,65 @@ +# How To: Extract a Component's Dist-Git Repository + +Use `azldev comp dist-git-repo` to extract one component's committed rendered +files and their Git history from the local project into a standalone repository. +This works for both locked and lockless components. + +Install **git-filter-repo** first. If `git filter-repo --version` is unavailable, +the command fails with an installation message before replacing any output. + +## Extract a component + +Specify exactly one component name and a required output directory: + +```bash +azldev comp dist-git-repo curl -o ./dist-git/curl +``` + +`--output-dir` is the long form of `-o`. Relative output paths are relative to +your current working directory. To select a project elsewhere, use the global +`-C` option: + +```bash +azldev -C /path/to/project comp dist-git-repo curl -o ./dist-git/curl +``` + +The command clones the local project repository's current branch using +`git clone --single-branch --no-local`. It then runs +`git filter-repo --subdirectory-filter` in the clone, selecting: + +```text +// +``` + +The path passed to the filter is relative to the project repository's Git root. +For example, `project.rendered-specs-dir = "specs"` selects `specs/c/curl`; +`SLOF` selects `specs/s/SLOF`. A configured nested rendered-specs directory is +respected. + +The selected files become the standalone repository's root: + +```bash +git -C ./dist-git/curl log --oneline +ls ./dist-git/curl +``` + +Only committed content is exported. This command does not render files, refresh pins, +or fetch the component's upstream Fedora/dist-git repository. Render and commit +changes first if you want them included. + +## Replace an existing output + +Even an empty existing directory is rejected unless you provide `--force`: + +```bash +azldev comp dist-git-repo curl --output-dir ./dist-git/curl --force +``` + +**`--force` deletes the existing output directory and its contents.** Prerequisite, +source-path, and output-safety checks run before deletion. The source +repository and home directory (including their ancestors), Git metadata, and paths +containing symbolic links are not allowed as destinations. + +If cloning or filtering fails, any partial output remains for inspection. +Resolve the cause and retry with `--force`. A global `--dry-run` makes no +filesystem changes and does not run Git commands. diff --git a/docs/user/how-to/preview-without-lockfile.md b/docs/user/how-to/preview-without-lockfile.md index 15332b637..dd614a06c 100644 --- a/docs/user/how-to/preview-without-lockfile.md +++ b/docs/user/how-to/preview-without-lockfile.md @@ -1,41 +1,170 @@ # How To: Preview the Lock-File-Free Mode -`--without-lockfile` is a **preview** global flag. It selects an alternative way of -tracking a component's resolved upstream commit: instead of per-component lock -files, azldev records the commit in generated component TOML that the project -includes like any other config file. +The **preview** lockless workflow tracks a component's resolved upstream commit +in generated component TOML instead of per-component lock files. Select it for +individual components using `lockless-components.txt`. To use it for the whole +project, list every component. -The flag is opt-in and defaults to off. Without it, azldev behaves exactly as it -always has — lock files, `component update`, `component history`, and -`component query` are unchanged. Nothing in the preview mode is stable yet; both -the command surface and the generated file layout may change. +The file applies to every command, not just rendering. Nothing in the preview +workflow is stable yet; both the command surface and generated layout may change. ```bash -# Default behavior: lock files. +# Use curl's selected workflow. azldev component render -p curl -# Preview behavior: generated upstream-commit config. -azldev --without-lockfile component render -p curl +# Refresh curl's generated commit if curl is listed in lockless-components.txt. +azldev component refresh-upstream-commit -p curl ``` -Pass the flag on every invocation that should use the preview mode, before the -command name. `--without-lockfile=false` explicitly selects the default mode. +## Select Lock Mode Per Component + +Create `lockless-components.txt` beside the top-level `azldev.toml`. List one +exact, case-sensitive component name per line: + +```text +curl +rust-asyncband +``` + +Rendering, builds, and source preparation ignore existing locks and lock +validation for listed components. They use configured upstream commits. +Lockless rendering, builds, queries, and source preparation do not generate +synthetic history; builds and queries use committed rendered history. +Unlisted components use lock files, lock validation, and fingerprint-derived +synthetic history. Both kinds can be processed together. +Locked rendering preserves existing non-symlink entries at percent-encoded alias +paths and warns about the collision; lockless rendering replaces those entries. +Locked rendered output is copied, so the work and output directories may reside +on different filesystems. Lockless output continues to use moves. +Only lockless `render --check-only` ignores root `.git` and +`.no-change-rebuild` entries. Locked rendering compares all existing output +entries after removing Git metadata from its staging directory. +Lockless Release edits preserve existing tag casing and alignment; locked +components retain the established `Release: value` normalization. + +A missing or empty file means all components use lock files. Blank lines and +surrounding whitespace are ignored; names are not glob patterns. A file read +error stops configuration loading. + +To add names and refresh their upstream commits in one command, first configure +the generated TOML include described below, then run: + +```bash +azldev comp make-lockless curl rust-asyncband +``` + +`make-lockless` accepts the normal component selectors (`-p`, `-g`, and `-a`). +It merges selected exact names into the top-level list and writes the complete +list in lexical order, one name per line without blank lines or duplicates. +Existing names are retained and lock files are left untouched. It then runs the +`refresh-upstream-commit` pipeline for the selection. +Already-listed components are refreshed again without adding +duplicates. Non-upstream components are listed too; refresh removes any generated +pin for them without contacting an upstream provider. +If refreshing fails, the list stays updated and successful pins are saved. +Fix the failure and rerun the command. Dry runs change neither the list nor pins. + +Both `component update` and `component refresh-upstream-commit` are available. +Update (including `--bump`) skips listed components and leaves +their locks untouched; refresh skips unlisted components and preserves their +generated TOML files. All-component scans still prune files for components that +no longer exist, not files belonging to components using the other mode. +This preservation includes generated pins for unlisted local components, both +when refreshing normally and when checking for stale files with `--check-only`. +`component history` skips listed components. `component query` extracts their +committed rendered dist-git history and processes it with rpmautospec before +querying; uncommitted rendered changes are ignored. `component changed` compares resolved project +inputs and committed rendered output for listed components, and stored fingerprints for unlisted components; +the current checkout's list determines which comparison to use. ## What Changes -| Area | Default | `--without-lockfile` | +| Area | Locked component | Lockless component | |------|---------|----------------------| | Resolved commit storage | `locks/.lock` | `base/upstream-commits/.toml` | | Refresh command | `azldev component update` | `azldev component refresh-upstream-commit` | -| Inspecting resolved state | `azldev component history`, `azldev component query` | read the generated TOML; no equivalent commands | -| Lock consistency checks | On, with `--skip-lock-validation` to opt out | Not applicable; the flag is not registered | -| `component changed` | Compares stored input fingerprints | Compares project configuration resolved at each ref | -| Synthetic dist-git history | Derived from lock-file fingerprint changes | Derived from generated upstream-commit TOML changes | -| Agent skills and MCP tools | Describe the lock-file workflow | Describe the upstream-commit workflow | +| Inspecting resolved state | `azldev component history`, `azldev component query` | `component query` inspects committed rendered specs; `history` skips lockless components | +| Lock consistency checks | On, with `--skip-lock-validation` to opt out | Skipped | +| `component changed` | Compares stored input fingerprints | Compares resolved build inputs and committed rendered output at each ref | +| Synthetic dist-git history | Derived from lock-file fingerprint changes | Not generated; build and query extract committed rendered history | +| Agent skills and MCP tools | Both workflows are available | Both workflows are available | + +### Preparing, querying, and building committed dist-git + +Render and commit the component before querying or building it. Lockless +`component query` and `component build` use the same extraction as +`component dist-git-repo`: a temporary `--single-branch --no-local` clone of the +current project branch, filtered with `git filter-repo --subdirectory-filter` +using the component's repository-relative rendered directory. Install +`git-filter-repo` on the host. Uncommitted rendered changes are not included, +and extraction never modifies the original repository. + +For a bulk query, the project branch is cloned once into a temporary bare +snapshot. Component repositories initially share that immutable object store +and are filtered with bounded CPU concurrency. Each filtered repository is then +repacked and detached from the shared store before entering Mock, so it contains +only its own history and does not depend on host-only object paths. + +`component query` processes the temporary spec with rpmautospec in the project's +Mock chroot, then reports its source NEVR, required files, and binary subpackages. +The temporary query repositories are removed on success and failure. +Locked queries continue to read working-tree rendered files, including uncommitted edits. + +`component build` fetches configured source files through the normal source manager, +including custom generation. Download-origin fallback is enabled unless +`disable-origins` is set. It then downloads any remaining payloads listed in the extracted +`sources` manifest from lookaside and builds the SRPM with Mock's `rpmautospec` +plugin enabled and the extracted repository passed as `--sources`. This requires Mock 5.3 or newer and +rpmautospec available from the build distro's package repositories. +Mock's rpmautospec plugin must be installed on the host as well; enabling an +unavailable plugin is an error, even if Mock itself would only warn. +The plugin resolves `%autorelease` and `%autochangelog` using the filtered +history; no synthetic history, overlay replay, or separate Release bump runs. +Build macros come from the committed spec, not duplicate +Mock `--define`, `--with`, or `--without` arguments. Render and commit macro changes +before building; live configuration cannot override committed macro settings. +Lockless rendering emits sorted `%define` entries at the top of the spec instead +of a separate `.azl.macros` file, `%{load:...}` directive, or generated `Source9999`. +These lazy definitions preserve references to macros defined later in the spec. +Locked components retain the separate macros file and its source entry. +Binary RPMs are built from that SRPM normally. `--no-check` is forwarded through +Mock's `--nocheck`; `--srpm-only` skips binary builds. Lockless builds reject +`--without-git` and `--rpmdev-bumpspec`. Locked build behavior is unchanged. -`component update`, `component history`, and `component query` remain registered -in preview mode as hidden no-ops so that existing scripts report clearly that the -commands do nothing, rather than failing with "unknown command". +`component prepare-sources` uses lockless render preparation while writing +directly into `--output-dir`, allowing uncommitted and untracked project files. +Current TOML configuration (including upstream pins and distro settings), overlay +inputs, scripts, and local sources are read from the working tree. Modified and +newly added files are used without committing them first. Only the existing +component dist-git state used as the release/changelog baseline is read from +committed files; modified and untracked files in that directory are ignored. +Nonempty output requires `--force`, which removes existing contents before +preparation. Failures can leave partially prepared output. It fetches upstream +lookaside payloads with the spec/sidecars, prepares the upstream release/changelog, +then downloads/generates configured `source-files`. Like locked preparation, it +acquires payloads before emitting build macros and applying overlays. Archive +overlays run first, followed by inline macro definitions, configured spec/loose-file +overlays, and generated check-skip/header overlays. It updates `sources` metadata +with the configured post-overlay hashes. It does not compare +rendered output, apply a final render release bump, or commit changes. +Render history is read from +`//` inside the +project repository, independently of `--output-dir`. The configured root defaults +to `specs/`. This path never generates synthetic history. Fetched upstream Git +metadata is retained through preparation; `--without-git` removes `.git` after +successful preparation. On failure, any remaining Git metadata is left for inspection. + +The source-preparation flags also apply to lockless components: + +| Flag | Behavior | +|------|----------| +| `--without-git` | Remove `.git` from prepared output after successful preparation. It remains available for upstream release/changelog preparation; no synthetic history is generated with or without the flag. | +| `--skip-overlays` | Skip configured and generated overlays, inline macro definitions, and `sources` metadata updates. Source acquisition still runs unless `--skip-sources` is also set. Upstream release/changelog preparation still runs. | +| `--skip-sources` | Skip upstream lookaside downloads, configured source downloads/generation, and archive overlays. Spec/sidecars are still fetched; other overlays and `sources` metadata updates still run unless `--skip-overlays` is set. Overlays requiring missing payloads can fail. | +| `--allow-no-hashes` | Compute missing hashes from acquired files when updating `sources`, defaulting to SHA-512 when `hash-type` is omitted. This does not edit TOML. With `--skip-sources`, missing hashes cannot be computed and cause an error; with `--skip-overlays`, metadata updates and hash computation are skipped. | + +Use `azldev component dist-git-repo -o ` to obtain a committed +working dist-git repository. That extraction command does not download archives. ## Configure the Project @@ -50,24 +179,37 @@ includes = [ ``` Generated files hold only `spec.upstream-commit`; the component's own TOML -supplies the source type and everything else. Because a single file may hold a -partial component definition in this mode, component validation runs after all -config files have been merged. +supplies the source type and everything else. Newly written files use one table, +`[components..spec]`, without separate parent-table headers: + +```toml +[components.curl.spec] +upstream-commit = 'abcdef1234567' +``` + +Explicit upstream pins may refer to commits on other branches. Source preparation +fetches all branches when an effective commit pin is present; floating lockless +sources retain the single-branch clone optimization. + +Existing files with explicit parent tables remain readable. Because a single file +may hold a partial component definition for a lockless component, its validation runs after +all config files have been merged. Unlisted components retain normal additive +merging and per-file validation. -An existing `[project] lock-dir` setting is accepted and ignored in preview mode, -so the same project config works in both modes. +`[project] lock-dir` still configures the directory used for locked components. +Lockless components do not read or write their lock files. ## Refresh a Component ```bash # Resolve and record the upstream commit for one component. -azldev --without-lockfile component refresh-upstream-commit -p curl +azldev component refresh-upstream-commit -p curl # Refresh everything and prune generated files for components that no longer exist. -azldev --without-lockfile component refresh-upstream-commit -a +azldev component refresh-upstream-commit -a # CI gate: exit 1 when any generated file is out of date. -azldev --without-lockfile component refresh-upstream-commit -a --check-only -q +azldev component refresh-upstream-commit -a --check-only -q ``` Refresh after changing a commit pin, upstream distro or version, or snapshot. @@ -80,36 +222,369 @@ resolved components are created or updated before the command exits with an error. Failed components remain unchanged, and orphan pruning is skipped for that run. -Commit the refreshed TOML together with the rendered output: synthetic dist-git -history — and therefore `%autorelease` and `%autochangelog` expansion — is derived -from committed changes to the generated file. Unlike the default mode, there is no -fingerprint to compare the working tree against, so uncommitted changes do not -produce a synthetic commit. +Commit the refreshed TOML before rendering: rendering compares the upstream pins +from committed configuration. Lockless rendering neither generates synthetic +history nor expands `%autorelease` or `%autochangelog`. + +## Render and Commit a Component + +The following process applies to **all components selected as lockless**. +Local components skip upstream-history queries and release/changelog carryover; +their Release and changelog are never automatically modified, including with +`--rpmdev-bumpspec`. They copy local files except the TOML defining the local spec +path, retain other TOML sidecars, and participate in unchanged-render checks and +automatic commits. For local components, `--changelog-comment` supplies only the +Git commit message; without it, rendering collects the message before replacing +output. Local components with `--no-git-commit` never prompt. +All lockless rendering runs on the host without mock; +components using lock files retain their existing mock-based rendering. + +Before any upstream checkout or component preparation, rendering checks for staged +changes, unstaged tracked changes, and untracked files outside the configured +`project.rendered-specs-dir` and inside every selected component's dist-git +directory (`//`). +Changes elsewhere under `project.rendered-specs-dir`, including other components' +rendered output, are ignored. This scope uses the configured root (default `specs/`), +not a command-line `--output-dir`. Git-ignored files do not block rendering. +If any selected component is lockless, changes in the checked paths stop the whole invocation. +Commit or remove those changes first, or explicitly bypass this check: + +```sh +azldev comp render -p --allow-uncommitted +``` + +`--check-only` and `--no-git-commit` still require the checked paths to be clean unless +`--allow-uncommitted` is specified. The override bypasses only the cleanliness +check: carryover and upstream-change comparison still use committed history, +and failure recovery can discard uncommitted files in the component output +directory. Locked components do not trigger this check. + +For unpinned upstream components, the renderer clones only the branch selected by +the component's effective `spec.upstream-distro` (or the project default) into +temporary storage. The resolved distro version's `dist-git-branch` supplies the +branch name. The clone retains that branch's full history so snapshot selection +remains available without a shallow-history limit. Pinned components fetch all +branches so commits outside the selected branch remain usable. This branch +restriction is specific to unpinned lockless preparation; locked components +retain normal multi-branch cloning. + +Immediately after checking out the pinned upstream commit, rendering captures +optional Fedora provenance values for `build.emit-upstream-provenance`, before +release/changelog carryover or bumping changes the spec. If the upstream release +uses `%autorelease` and `build.defines` does not already provide +`fedora_upstream_release`, it runs host `rpmautospec calculate-release` at this +point. The captured values are reused later when emitting inline macros, without +repeating the query against the modified checkout. No provenance query runs when +provenance is disabled or the component is not sourced from Fedora upstream. + +The renderer then finds the existing committed output under +`//`, ignoring +uncommitted spec changes. It prepares the changelog first, then Release, before +emitting inline macros, applying overlays, or updating `sources`. + +Changelog handling is independent of version changes. For non-`%autorelease` +specs, the committed inline `%changelog` section is preserved. For an initial +`%autorelease` render, host `rpmautospec generate-changelog` creates the complete +upstream `changelog`, replacing any existing file. For legacy-marker output, the +working `changelog` is replaced with the committed inline `%changelog` body, +excluding the section header and leading or trailing comment lines starting with +`##`; interior comments and whitespace are preserved. For modern `%autorelease` +output, the committed `changelog` file replaces the working file byte-for-byte, +or is created if absent from the working checkout. A missing committed +`changelog` fails rendering. + +Release preparation reads the upstream spec's `Release:` and `Version:` before +overlays, and compares that version with the committed spec's +`Version:`, independently of upstream pin changes. Static release carryover runs +**only when committed output exists and the version is unchanged**; otherwise the +working static release and `baserelease` definition are left as supplied by +upstream and overlays. Autorelease follows the case-specific rules below. +There is no later restoration of the original upstream release. + +A spec may define several `Release:` tags, such as one per `%if`/`%else` +branch. Each tag is updated separately and keeps its own value and options. +All `Release:` tags in a spec must either use `%autorelease` or be static; +mixing the two fails rendering. Static carryover copies each committed +`Release:` value to the upstream tag at the same position, and fails if the +upstream and committed specs have different numbers of `Release:` tags. When +upstream switches from `%autorelease` to a static release, every committed +`%autorelease` tag must have the same value. + +When release carryover runs for an existing non-`%autorelease` upstream spec, it +checks the committed `Release:` value. If it references the `baserelease` macro, +the renderer copies the committed `%global baserelease` or `%define baserelease` +line over the upstream definition, leaving the upstream `Release:` value +unchanged. A missing or ambiguous macro definition fails rendering. Otherwise, +it copies the committed `Release:` value into the upstream spec's `Release:`. +No temporary release-bump probe is performed. + +When upstream switches from `%autorelease` to a static release, the renderer +first extracts the committed component history and calculates its complete +release with host rpmautospec. For an unchanged version, that concrete release +is the baseline for the normal static bump; the old `%autorelease` macro is +never restored. An existing `%autochangelog` is materialized from the same +history before adding the static changelog entry. This transition requires +`git-filter-repo`; a changed version still starts from the new upstream release. + +For an initial `%autorelease` render, it calculates the numeric upstream release +and adds or replaces the macro's `-b` parameter. This rpmautospec operation runs +on the host; it does not preprocess or rewrite the spec. Initial and legacy +autorelease handling do not depend on version changes. + +For an existing output whose spec has a line starting with `## RPMAUTOSPEC:` +and containing `autorelease` on that same line, a `%autorelease` upstream spec +inherits its base from the committed rendered spec. The renderer expands the +committed `Release:` with host `rpmspec`, using a temporary copy of the +committed dist-git files, and uses its leading integer as `-b` (for example, +`9.fc43` becomes `-b 9`). + +For an existing `%autorelease` spec without the legacy marker, rendering carries +the committed `-b` value into the working spec **only when the version is +unchanged**, preserving the working macro's other options. If the committed spec +omits `-b`, the working spec also omits it, removing any working base option +rather than adding `-b 1`. An invalid numeric base fails release carryover. +No host release query is needed for unchanged-version carryover. When the version +changes, rendering calculates the upstream numeric base on the host and adds or +replaces `-b`, just as for an initial render. This changes only release preparation: +the committed changelog is still carried over as described below, not regenerated. + +For calculated bases, including legacy expansion, a base of `1` omits `-b` +and removes any existing working base option, using the `%autorelease` default. +Other macro options are preserved. Unchanged-version modern output still carries +an explicitly committed `-b 1` as-is. + +The last commit touching the component output is the **last render commit**. +The renderer fully loads TOML, including includes and component defaults, at +that commit and at the current project commit to compare `spec.upstream-commit`. +If the last render has no configured pin, its committed lock file supplies +`upstream-commit` when that commit's top-level `lockless-components.txt` is absent +or does not list the component. The historical configuration's +`lock-dir` determines which lock file to read; current or uncommitted locks are +never used for this fallback. +When the last render used a different upstream branch and the pins differ, the +renderer fetches that branch into the working clone before comparing commits. +Its branch is resolved from the last render's committed TOML, including includes, +component overrides, and distro defaults. The current branch remains available; +fetching the historical branch does not change the working checkout. +Different pins, missing render history, or an error loading the old pin mean +upstream changed. It records `git log --oneline ..` from the +upstream checkout; without an old pin it records a one-line version update. +If the old pin cannot be resolved in the single-branch checkout, rendering +fetches that specific commit hash from `origin` and checks again. If the commit +remains unavailable, including after a failed historical-branch fetch, rendering +continues with `update to upstream commit ` as the +upstream changes text. Other errors collecting the Git log remain fatal. +An error reading the current pin is fatal. + +It then compares the prepared tree with existing output, ignoring the root `.git` +and any top-level `.no-change-rebuild` on both sides. +`--check-only` stops here, before the final release bump. +If normal rendering detects no changes, it returns an error without replacing +the existing output or writing a failure marker. Unchanged local non-autorelease +components must manually update their release and changelog; they fail even with +`--no-change-rebuild`. To explicitly rebuild unchanged upstream content or a local +autorelease component, +pass `--no-change-rebuild`, a boolean switch taking no argument: + +```bash +azldev comp render -p curl --no-change-rebuild +``` + +For an unchanged autorelease component (upstream or local) with this flag, the +tool runs host `date -Is` and writes its exact output to `.no-change-rebuild` at +the working dist-git root. The marker is then published and committed with the +rendered files. Unchanged upstream non-autorelease components instead proceed +to the normal release/changelog bump without writing a timestamp marker. +Local specs are never automatically bumped. +The flag does not create a marker when comparison detects real changes. +The root marker is ignored in both the working upstream checkout and existing +output, even if its contents differ. Nested files with the same name are compared. + +Changed renders, or explicitly requested no-change rebuilds, then invoke host +`rpmdev-bumpspec -c` **only for upstream non-autorelease specs**, with the exact +`--changelog-comment` value. There is no default comment. If the flag is omitted, +rendering collects any needed message **after comparison and before bumping or +replacing output**, using `$EDITOR` (including configured arguments), or a local +editor (`editor`, `sensible-editor`, `vi`, or `nano`). The editor +starts with two blank lines followed by `#`-commented instructions when there are +no upstream changes. Otherwise, the first line is blank for your summary, and +each upstream-change line is prefilled with a `- ` prefix. Non-autorelease prompts +start those changes on line 2. Autorelease prompts put `...` on line 2 and start +the changes on line 3. These are editable message text, not instructional comments. +Save and close the editor to continue. Blank and whitespace-only lines and lines starting with +`#` are removed; other message lines are preserved, with no trailing newline. +The cleaned text is passed to `rpmdev-bumpspec -c` and reused as the Git commit +message, separating its first line (the Git summary) from subsequent body lines +with a blank line. Bumpspec leaves a single blank line between the new and +previous changelog entries. Explicit `--changelog-comment` values remain unchanged. +An edited message containing only ignored lines or editor failure aborts rendering +before output replacement. Missing-comment finishing is serialized to prevent +simultaneous editors. + +Local components and autorelease specs skip `rpmdev-bumpspec` entirely. +For upstream non-autorelease specs, a temporary process-local RPM wrapper sets +`_sourcedir` and `_specdir` to the spec's directory so sidecar macros resolve +correctly. Rendering preserves your `HOME`, packager settings, locale, and +timezone; it does not edit your `.rpmmacros`. The wrapper is removed after the +invocation, including when bumpspec fails. `Release:` lines inside `%changelog` +text (for example, quoted spec snippets) are temporarily masked so bumpspec +changes only real `Release` tags; they are restored unchanged afterward. + +When `--changelog-comment` is omitted, the prompt depends on how the message is used: + +| Component | Automatic Git commit | `--no-git-commit` | +|-----------|----------------------|-------------------| +| Upstream non-autorelease | Changelog and commit message | Changelog message only | +| Autorelease (upstream or local) | First-line changelog/commit summary, then commit body | No prompt | +| Local non-autorelease | Git commit message only; changelog stays unchanged | No prompt | + +Each case has its own instructional comments. The final Git commit always reuses +the supplied or already collected message; it never opens another editor. +`--check-only` and dry runs never open editors. +For unattended rendering, provide `--changelog-comment`. +Any required editor prompt first checks that both stdin and stdout are TTYs. +Otherwise rendering fails before opening an editor or replacing output: +non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +For upstream non-autorelease components, `--no-git-commit` alone is insufficient: +the changelog bump still requires `--changelog-comment`. +Prefilled upstream changes are used only as retained in the edited message; +the original changes are not appended again to its changelog entry or Git body. + +```bash +azldev comp render -p curl --changelog-comment "Fix curl patches" +``` + +Macro-file generation, existing overlays, and `sources` manifest updates run +after changelog and Release preparation, but before comparison and the final +`rpmdev-bumpspec` invocation. Overlays can override the prepared Release and +changelog; carryover is not repeated afterwards. Version changes made by overlays +do not alter the earlier carryover decision. +Release edits preserve the existing spaces and tabs between `Release:` and its +value, including during overlays and carryover, so aligned spec headers stay aligned +with either spec editor. +Rendered output retains all prepared dist-git files except `.git`, including +unreferenced sidecars, hidden files, and subdirectories. Local spec-defining TOML +files are excluded during copying. +Lockless rendering does not run `spectool` or filter files; `render.skip-file-filter` +has no effect on lockless components. Synthetic history and its legacy release +adjustment are skipped. The spec is **not** preprocessed with `rpmautospec process-distgit`: +`%autorelease`, `%autochangelog`, and other RPM macros remain unexpanded. +No mock configuration is needed for lockless rendering. +Comparison ignores the root `.git` entry without traversing or removing it; +the working checkout retains its Git metadata until staging cleanup. +After comparison and the final release bump, but immediately before removing +existing output, lockless rendering queries the existing and new source-package +NEVRs with host `rpmspec`. Queries expand each spec's macros using its own dist-git +directory. A spec whose `Release` uses `%autorelease` without having been processed +by rpmautospec (no `## RPMAUTOSPEC` header) is instead resolved as lockless builds +resolve it: against the component's filtered rendered-directory history, as +produced by `git filter-repo --subdirectory-filter`, using `rpmautospec +process-distgit` before the `rpmspec` query. The existing spec uses the committed +history; the new spec adds the prepared output as the next commit, unless its tree +is unchanged. For merge-free project history, the filtered history is replayed in a +temporary repository that reads project objects through Git alternates and never +writes to the project repository; histories with merges are cloned and filtered +with `git filter-repo`, which must then be installed. Plain `rpmspec` would expand +`%autorelease -b N` to `N`, ignoring the commits that raise the built release. +The package name must match. When the final new spec uses +`%autorelease`, its new epoch/version/release must be equal to or higher than the +existing value under RPM version ordering. Non-autorelease specs still require +a strictly higher value. Autorelease permits equality because unchanged output +creates no new commit. Lower NEVRs, and equal non-autorelease NEVRs, fail with both values +in the error message; query failures also abort. These errors return +a failing exit status even without `--fail-on-error`, and leave existing output +untouched. First renders with no existing directory skip the check. +`--check-only` and dry runs do not perform this replacement-time check. +`--no-change-rebuild` does not bypass it: a timestamp-only autorelease rebuild +may have an equal expanded NEVR, but must not decrease it. + +After successful validation, lockless rendering removes the +existing output directory and moves all prepared entries except `.git` into +its replacement. Files found only in the old output are not retained. +Moves use filesystem renames, so the project work directory and rendered +output must be on the same filesystem. +The renderer then commits the component output directory noninteractively, using +the supplied or already edited message. Its first line is the Git summary; +remaining lines form the body. Prefilled upstream changes appear only as retained +in the edited message; explicitly supplied flag messages keep the original +upstream changes in a separate Git body paragraph. +The first successful automatic commit also +includes all staged changes, unstaged tracked changes, and non-ignored untracked +files outside `rendered-specs-dir` (or the active `--output-dir`). Other component +directories and unrelated changes within the rendered output directory are +excluded from that commit, even if already staged. Later component commits +include only their own dist-git output. +Unchanged Git content does not produce an empty commit. +Pass `--no-git-commit` to skip automatic staging and committing while still +rendering and bumping non-autorelease specs. The rendered output remains available for review +and manual commit. These flags affect all lockless components; +locked rendering retains its existing behavior. Local Release/changelog are not bumped. + +```bash +azldev comp render -p curl --changelog-comment "Fix curl patches" --no-git-commit +``` + +Lockless component failures never create `RENDER_FAILED` or replace existing +output with a failure marker. Failures before output replacement leave the +existing directory unchanged. If replacement or automatic committing fails after +removal begins, rendering clears the component output and resets only that +component's staged paths. If output existed, Git checks out its committed +contents from `HEAD`; otherwise, the directory remains absent. +This recovery does not preserve uncommitted changes or untracked files in the +replaced component directory, and uses no backup or index snapshot. Recovery +errors are reported with the original render failure. +Files outside the rendered output directory may remain staged if automatic +committing fails; component recovery does not reset those paths. +`--check-only` failures leave output untouched and fail the check. +Locked rendering retains its failure marker behavior. + +For component names requiring URL encoding, rendering also creates a sibling +alias such as `libxml%2B%2B -> libxml++` for downstream tooling. Any existing +entry at the encoded alias path is removed first, including regular files and +nonempty directories, and replaced with a relative symlink. Existing symlink +targets are not removed. Alias creation failures produce a warning. +Lockless aliases are updated only after rendering and any automatic commit succeed. +`--check-only` does not replace aliases. + +Install `rpmdevtools`, `rpm-build`, and `rpmautospec` on the host as needed for +the release/changelog queries. Output (including a custom `--output-dir`) must be inside the project +Git repository. Git author identity is needed for automatic commits; it is not +required by the renderer with `--no-git-commit`. +`--check-only` prepares and compares temporary output but never +performs the final `rpmdev-bumpspec` invocation, replaces rendered files, or +creates project commits. `--check-only` also never creates a rebuild timestamp, +even with `--no-change-rebuild`. The no-change refusal and rebuild flag apply only +to all lockless components; locked components retain their existing +behavior. Change reporting includes a forced rebuild marker and changes made by +the final bump. ## Detect Changed Components ```bash -azldev --without-lockfile component changed --from main -a -q -O json +azldev component changed --from main -a -q -O json ``` -In preview mode this loads the project configuration independently at both refs +For lockless components this loads the project configuration independently at both refs and compares the resolved component build inputs: normalized component configuration, upstream commit or local spec-directory contents, overlay source filenames and contents, and the effective distro release version. Documentation, publishing, test-selection, scheduling-hint, snapshot-time, and checkout-path-only fields do not mark a component as changed. +It also compares the committed rendered component directories, including specs, +sidecars, and `.no-change-rebuild` markers. Render-only commits therefore select +the component for rebuilding even when its configuration and upstream pin are +unchanged. `sources-change` still reports only changes to the `sources` manifest. ## Emit Agent Files for the Preview Mode -`azldev docs agent install` emits the content for the mode it runs in, so pass the -flag when the target repository uses the preview workflow: +`azldev docs agent install` includes both workflows, selected per component by +the project's list: ```bash -azldev --without-lockfile docs agent install +azldev docs agent install ``` ## Reference Documentation The generated CLI reference under [reference/cli/](../reference/cli/azldev.md) -documents azldev's default mode. Use `azldev --without-lockfile --help` -to see the preview mode's command surface and help text. +documents both workflows. Use `azldev --help` for command syntax. diff --git a/docs/user/reference/cli/azldev.md b/docs/user/reference/cli/azldev.md index c5de33703..84dd91958 100644 --- a/docs/user/reference/cli/azldev.md +++ b/docs/user/reference/cli/azldev.md @@ -30,7 +30,6 @@ lives), or use -C to point to one. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced.md b/docs/user/reference/cli/azldev_advanced.md index f76116d52..2903a74ef 100644 --- a/docs/user/reference/cli/azldev_advanced.md +++ b/docs/user/reference/cli/azldev_advanced.md @@ -33,7 +33,6 @@ output but fully supported. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_download-sources.md b/docs/user/reference/cli/azldev_advanced_download-sources.md index 7758ecd08..cabe219da 100644 --- a/docs/user/reference/cli/azldev_advanced_download-sources.md +++ b/docs/user/reference/cli/azldev_advanced_download-sources.md @@ -71,7 +71,6 @@ azldev advanced download-sources [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mcp.md b/docs/user/reference/cli/azldev_advanced_mcp.md index 1f62c5e0c..abf04adaa 100644 --- a/docs/user/reference/cli/azldev_advanced_mcp.md +++ b/docs/user/reference/cli/azldev_advanced_mcp.md @@ -44,7 +44,6 @@ azldev advanced mcp [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock.md b/docs/user/reference/cli/azldev_advanced_mock.md index 984278eea..849e14746 100644 --- a/docs/user/reference/cli/azldev_advanced_mock.md +++ b/docs/user/reference/cli/azldev_advanced_mock.md @@ -32,7 +32,6 @@ starting interactive shell sessions in mock chroot environments. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md b/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md index 18e1be1a3..9068693c3 100644 --- a/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md +++ b/docs/user/reference/cli/azldev_advanced_mock_build-rpms.md @@ -51,7 +51,6 @@ azldev advanced mock build-rpms [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_mock_shell.md b/docs/user/reference/cli/azldev_advanced_mock_shell.md index 54d6ebc76..7c5bf9e28 100644 --- a/docs/user/reference/cli/azldev_advanced_mock_shell.md +++ b/docs/user/reference/cli/azldev_advanced_mock_shell.md @@ -56,7 +56,6 @@ azldev advanced mock shell [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_advanced_wget.md b/docs/user/reference/cli/azldev_advanced_wget.md index aa23ba04a..cd5623b7a 100644 --- a/docs/user/reference/cli/azldev_advanced_wget.md +++ b/docs/user/reference/cli/azldev_advanced_wget.md @@ -45,7 +45,6 @@ azldev advanced wget URI [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion.md b/docs/user/reference/cli/azldev_completion.md index 109d7858e..6ec224917 100644 --- a/docs/user/reference/cli/azldev_completion.md +++ b/docs/user/reference/cli/azldev_completion.md @@ -31,7 +31,6 @@ See each sub-command's help for details on how to use the generated script. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_bash.md b/docs/user/reference/cli/azldev_completion_bash.md index f39580feb..74fe45a56 100644 --- a/docs/user/reference/cli/azldev_completion_bash.md +++ b/docs/user/reference/cli/azldev_completion_bash.md @@ -54,7 +54,6 @@ azldev completion bash -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_fish.md b/docs/user/reference/cli/azldev_completion_fish.md index 42eaf1dd5..b2a03faa7 100644 --- a/docs/user/reference/cli/azldev_completion_fish.md +++ b/docs/user/reference/cli/azldev_completion_fish.md @@ -45,7 +45,6 @@ azldev completion fish [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_powershell.md b/docs/user/reference/cli/azldev_completion_powershell.md index 6565209e0..24ffeb519 100644 --- a/docs/user/reference/cli/azldev_completion_powershell.md +++ b/docs/user/reference/cli/azldev_completion_powershell.md @@ -42,7 +42,6 @@ azldev completion powershell [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_completion_zsh.md b/docs/user/reference/cli/azldev_completion_zsh.md index aa7d5d7a4..b1e9fa460 100644 --- a/docs/user/reference/cli/azldev_completion_zsh.md +++ b/docs/user/reference/cli/azldev_completion_zsh.md @@ -56,7 +56,6 @@ azldev completion zsh [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component.md b/docs/user/reference/cli/azldev_component.md index df0e9622b..e46d7a5db 100644 --- a/docs/user/reference/cli/azldev_component.md +++ b/docs/user/reference/cli/azldev_component.md @@ -35,7 +35,6 @@ components defined in the project configuration. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO @@ -45,11 +44,14 @@ components defined in the project configuration. * [azldev component build](azldev_component_build.md) - Build packages for components * [azldev component changed](azldev_component_changed.md) - Detect which components changed between two git refs * [azldev component diff-sources](azldev_component_diff-sources.md) - Show the diff that overlays apply to a component's sources +* [azldev component dist-git-repo](azldev_component_dist-git-repo.md) - Extract a component's committed dist-git history into a repository * [azldev component history](azldev_component_history.md) - Report per-component change activity and customization detail * [azldev component list](azldev_component_list.md) - List components in this project +* [azldev component make-lockless](azldev_component_make-lockless.md) - Select the lockless workflow and refresh upstream commits * [azldev component metadata](azldev_component_metadata.md) - List documentation metadata for components' overlays and groups * [azldev component prepare-sources](azldev_component_prepare-sources.md) - Prepare buildable sources for components * [azldev component query](azldev_component_query.md) - Query info from locally rendered component specs +* [azldev component refresh-upstream-commit](azldev_component_refresh-upstream-commit.md) - Resolve and record upstream commits for components * [azldev component render](azldev_component_render.md) - Render post-overlay specs and sidecar files to a checked-in directory * [azldev component test](azldev_component_test.md) - Run a component's TMT tests in a local QEMU VM * [azldev component update](azldev_component_update.md) - Resolve and lock source identities for components diff --git a/docs/user/reference/cli/azldev_component_add.md b/docs/user/reference/cli/azldev_component_add.md index 3b0b66fd0..53fb56931 100644 --- a/docs/user/reference/cli/azldev_component_add.md +++ b/docs/user/reference/cli/azldev_component_add.md @@ -48,7 +48,6 @@ azldev component add [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_build.md b/docs/user/reference/cli/azldev_component_build.md index 278ab3f6c..4a12dd2de 100644 --- a/docs/user/reference/cli/azldev_component_build.md +++ b/docs/user/reference/cli/azldev_component_build.md @@ -16,6 +16,12 @@ are placed in structured subdirectories under the project output directory: out/rpms/ - binary RPMs with no channel configured (or channel="none") out/rpms// - binary RPMs moved to their configured publish channel subdirectory +For lockless components, build instead extracts committed rendered dist-git +history using git-filter-repo, downloads lookaside sources, and builds the SRPM +with mock's rpmautospec plugin (mock 5.3+). Uncommitted rendered changes are +ignored. '--without-git' and '--rpmdev-bumpspec' are unsupported for lockless +components. '--no-check' skips the binary build's package checks normally. + The publish channel for each package is resolved from the project's package configuration (package groups and per-component overrides). See 'azldev package' for details. @@ -80,7 +86,6 @@ azldev component build [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_changed.md b/docs/user/reference/cli/azldev_component_changed.md index 2752c4ea2..c7a585d4a 100644 --- a/docs/user/reference/cli/azldev_component_changed.md +++ b/docs/user/reference/cli/azldev_component_changed.md @@ -12,6 +12,10 @@ input fingerprint changed (any change). For components whose fingerprint changed, it also reports whether the rendered sources file changed (sources change). +For components listed in lockless-components.txt beside the current checkout's +top-level azldev.toml, compare resolved project build inputs at both refs instead +of stored locks. + This is useful for CI/CD pipelines to determine which components need to be rebuilt or have their lookaside tarballs re-uploaded after a PR merge. @@ -70,7 +74,6 @@ azldev component changed [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_diff-sources.md b/docs/user/reference/cli/azldev_component_diff-sources.md index 4bcb11e90..4617a56f7 100644 --- a/docs/user/reference/cli/azldev_component_diff-sources.md +++ b/docs/user/reference/cli/azldev_component_diff-sources.md @@ -42,7 +42,6 @@ azldev component diff-sources [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_dist-git-repo.md b/docs/user/reference/cli/azldev_component_dist-git-repo.md new file mode 100644 index 000000000..d9639ac53 --- /dev/null +++ b/docs/user/reference/cli/azldev_component_dist-git-repo.md @@ -0,0 +1,64 @@ + + +## azldev component dist-git-repo + +Extract a component's committed dist-git history into a repository + +### Synopsis + +Clone the local project repository's current branch with '--single-branch' +and '--no-local', then run 'git filter-repo --subdirectory-filter' to retain +only //. +The component's dist-git files become the new repository's root. + +Requires exactly one component name, '--output-dir' (-o), and git-filter-repo. +Only committed files and history are exported; this does not render the component +or clone its upstream distro repository. + +An existing output directory is rejected unless '--force' is supplied. +With '--force', that directory is deleted and recreated after preflight checks. +For safety, the source repository and home directory (including their ancestors), +Git metadata, and paths containing symbolic links cannot be used as output directories. +Clone or filtering failures leave any partial output available for inspection. + +``` +azldev component dist-git-repo [flags] +``` + +### Examples + +``` + azldev comp dist-git-repo curl -o ./dist-git/curl + azldev comp dist-git-repo rust-asyncband --output-dir ./dist-git/rust-asyncband --force +``` + +### Options + +``` + --force delete and recreate an existing output directory + -h, --help help for dist-git-repo + -o, --output-dir string destination for the extracted repository (required) +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --concurrency int base concurrency limit (default: number of logical CPUs; minimum 1) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev component](azldev_component.md) - Manage components + diff --git a/docs/user/reference/cli/azldev_component_history.md b/docs/user/reference/cli/azldev_component_history.md index b269a82d5..6dea6264b 100644 --- a/docs/user/reference/cli/azldev_component_history.md +++ b/docs/user/reference/cli/azldev_component_history.md @@ -15,6 +15,8 @@ Report three independent change-activity signals per component: Use this to find which packages get the most attention (for documentation, review prioritization, or refactoring planning). +Components using the lockless workflow via lockless-components.txt are skipped. + When a component shares its source TOML with other components (e.g., a bare entry in a shared components.toml), the toml-commit count is coarse and the component is marked 'toml-shared'. Use --shared=omit to drop those rows. @@ -68,7 +70,6 @@ azldev component history [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_list.md b/docs/user/reference/cli/azldev_component_list.md index f0cdcfd7c..448d1e123 100644 --- a/docs/user/reference/cli/azldev_component_list.md +++ b/docs/user/reference/cli/azldev_component_list.md @@ -57,7 +57,6 @@ azldev component list [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_make-lockless.md b/docs/user/reference/cli/azldev_component_make-lockless.md new file mode 100644 index 000000000..f9bafc00e --- /dev/null +++ b/docs/user/reference/cli/azldev_component_make-lockless.md @@ -0,0 +1,70 @@ + + +## azldev component make-lockless + +Select the lockless workflow and refresh upstream commits + +### Synopsis + +Add selected component names to lockless-components.txt beside the top-level +azldev.toml, then run the refresh-upstream-commit pipeline for those components. + +Names are written in lexical order, one exact component name per line, +without blank lines or duplicates. Existing names are retained; selection +patterns are expanded to exact component names. +Existing lock files are left untouched. Already-listed components are refreshed +again without duplicating their entries. + +Upstream components get generated commit TOML under base/upstream-commits/. +Include that directory in azldev.toml before component-specific configuration. +Non-upstream components are listed too, but do not contact upstream providers. +The refresh pipeline removes their generated commit TOML if one exists. + +If refreshing fails, the updated selection remains in place and successful +component pins are still saved. Fix the failure and rerun this command. + +``` +azldev component make-lockless [flags] +``` + +### Examples + +``` + azldev comp make-lockless -p curl + azldev comp make-lockless curl rust-asyncband + azldev comp make-lockless -g core + azldev comp make-lockless -a +``` + +### Options + +``` + -a, --all-components Include all components + -p, --component stringArray Component name pattern + -g, --component-group stringArray Component group name + -h, --help help for make-lockless + -s, --spec-path stringArray Spec path +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --concurrency int base concurrency limit (default: number of logical CPUs; minimum 1) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev component](azldev_component.md) - Manage components + diff --git a/docs/user/reference/cli/azldev_component_metadata.md b/docs/user/reference/cli/azldev_component_metadata.md index 35b0dc208..130b85186 100644 --- a/docs/user/reference/cli/azldev_component_metadata.md +++ b/docs/user/reference/cli/azldev_component_metadata.md @@ -83,7 +83,6 @@ azldev component metadata [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 45f9f8cb7..952325a08 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -14,6 +14,20 @@ for inspection or manual building. This is useful for verifying that overlays apply cleanly before running a full build. Only one component may be selected at a time. +For lockless components, render-style preparation writes directly to '--output-dir' +and preserves fetched upstream Git metadata. Nonempty output requires '--force'. +Current TOML and file inputs use the working tree, including modified and untracked +files. Release/changelog baselines use only committed component dist-git files; +uncommitted edits in that existing dist-git directory are ignored. +It downloads/generates source payloads before inline macro definitions and configured +overlays. Lockless specs contain %define entries, not a separate macros file, +load directive, or Source9999 entry. Archive overlays run before spec and loose-file overlays. It does not +compare rendered output, apply a final render release bump, or commit changes. +Render history comes from 'project.rendered-specs-dir'//, +independently of '--output-dir'. No synthetic history is generated. +'--without-git' removes fetched Git metadata after successful preparation. +It honors '--skip-overlays', '--skip-sources', and '--allow-no-hashes'. +Use 'azldev component dist-git-repo' to extract committed rendered history. ``` azldev component prepare-sources [flags] @@ -44,7 +58,7 @@ azldev component prepare-sources [flags] --skip-overlays skip applying overlays to prepared sources --skip-sources skip downloading fetched sources when preparing the package (useful to extract dist-git metadata when source files are not needed) -s, --spec-path stringArray Spec path - --without-git Disable dist-git repository creation (enabled by default) + --without-git Exclude Git metadata from prepared output (locked components also skip synthetic dist-git creation) ``` ### Options inherited from parent commands @@ -63,7 +77,6 @@ azldev component prepare-sources [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_query.md b/docs/user/reference/cli/azldev_component_query.md index f840ca4cd..b90411f47 100644 --- a/docs/user/reference/cli/azldev_component_query.md +++ b/docs/user/reference/cli/azldev_component_query.md @@ -10,11 +10,16 @@ Query detailed information for components from their locally rendered specs. This command reads the post-overlay specs from the project's rendered-specs-dir (produced by 'azldev component render') and runs rpmspec against them in a -single shared mock chroot, batching all specs into one chroot invocation with -parallel per-spec processing. For each component, it reports the source NEVR +single shared mock chroot, querying specs in batches with parallel per-spec +processing. For each component, it reports the source NEVR and the list of binary subpackages the spec would produce when built. -The rendered-specs-dir must exist on disk; if it doesn't, run +For lockless components, queries use committed rendered files from a temporary +dist-git repository extracted with git-filter-repo. rpmautospec resolves the +release and changelog in that temporary repository before querying. Uncommitted +rendered changes are ignored. Requires git-filter-repo for lockless components. + +For locked components, rendered-specs-dir must exist on disk; if it doesn't, run 'azldev component render' first. Components that previously failed to render (those with a RENDER_FAILED marker file) are skipped with a warning. @@ -60,7 +65,6 @@ azldev component query [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md b/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md new file mode 100644 index 000000000..f8467899e --- /dev/null +++ b/docs/user/reference/cli/azldev_component_refresh-upstream-commit.md @@ -0,0 +1,99 @@ + + +## azldev component refresh-upstream-commit + +Resolve and record upstream commits for components + +### Synopsis + +Resolve upstream commits for components and write normal per-component TOML configuration. + +For upstream components, this resolves the effective commit hash using the +distro snapshot time, then records it as spec.upstream-commit in +base/upstream-commits/.toml by default. Include that directory's TOML +files before component-specific TOML configuration so subsequent commands use +the generated commit unless the component configuration explicitly overrides +it. + +Only components listed in lockless-components.txt +beside the top-level azldev.toml are refreshed. Other components are skipped +and their generated TOML files are preserved. + +All TOML configuration is resolved before the source type is checked. Selected +components whose effective spec.type is not "upstream" do not contact an +upstream provider, and any generated upstream-commit TOML for them is removed. +Configuration is loaded permissively for this command so stale generated pins +cannot prevent cleanup after a component is removed or changed to another +source type. Other configuration validation failures are reported as warnings. + +If an upstream component fails to resolve and '--check-only' is not set, +successful components are still written before the command returns an error. +Failed components are left unchanged, and orphan pruning is skipped for that +run. + +When updating all components (-a), orphan generated TOML files are +automatically pruned. +Orphan pruning is skipped when updating individual components to avoid +accidentally removing files for components not included in the filter. + +The --check-only flag runs the full pipeline but does NOT write TOML files or +prune orphans. The command exits 0 when nothing would change and exits 1 when +any component is stale or any generated TOML would be pruned. Intended for CI gates. + +``` +azldev component refresh-upstream-commit [flags] +``` + +### Examples + +``` + # Refresh all components + azldev component refresh-upstream-commit -a + + # Refresh a single component + azldev component refresh-upstream-commit -p curl + + # Refresh components in a group + azldev component refresh-upstream-commit -g core + + # Write generated files to a custom directory + azldev component refresh-upstream-commit -a --upstream-commits-dir config/commits + + # CI gate: exit 0 if commit TOMLs are current, 1 if anything would change + azldev component refresh-upstream-commit -a --check-only -q +``` + +### Options + +``` + -a, --all-components Include all components + --check-only resolve upstream commits but do not write TOML files or prune orphans. Exits 0 when nothing would change and 1 when any component is stale (or, with --all-components, when any orphan generated TOML would be pruned). Intended for CI gates + -p, --component stringArray Component name pattern + -g, --component-group stringArray Component group name + -h, --help help for refresh-upstream-commit + -s, --spec-path stringArray Spec path + --upstream-commits-dir string directory for generated per-component upstream-commit TOML files (default "base/upstream-commits") +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --concurrency int base concurrency limit (default: number of logical CPUs; minimum 1) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev component](azldev_component.md) - Manage components + diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 88a9d5157..c17557e01 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -20,6 +20,45 @@ Unlike prepare-sources, render skips downloading source tarballs from the lookaside cache — only spec files, patches, scripts, and other git-tracked sidecar files are included. Multiple components can be rendered at once. +Render mode is selected per component by lockless-components.txt beside the +top-level azldev.toml. Each nonempty line names one component to render without +its lock file, using configured upstream commits and their TOML history instead. +All other components use lock files. A missing or empty file means all components +use lock files. + +For lockless upstream components, prepare the changelog, then Release, using the +upstream spec and last committed render before writing macros, applying overlays, +and updating sources. Overlays can override the prepared release and changelog. +Compare output; --no-change-rebuild permits unchanged upstream renders and records +a timestamp only for autorelease components (upstream or local). +Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. +Autorelease skips bumpspec. Missing messages are collected in an editor before +bumping or replacing output; the final Git commit never prompts. +Autorelease with --no-git-commit needs no message and never prompts. +Message prompts require both stdin and stdout to be interactive TTYs. +Before replacing existing lockless output, compare NEVRs using host rpmspec. +Autorelease requires an equal or higher NEVR; non-autorelease requires a higher NEVR. +The first commit also includes all pending changes outside the rendered output directory. +Later commits are component-only. --no-git-commit skips staging and committing. +Output must be inside the project repository; configure Git author identity and +install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. +--check-only stops before bumping, replacing output, or committing. +Lockless rendering never uses mock or rpmautospec process-distgit. +The spec retains its RPM macros, including %autorelease and %autochangelog. +All prepared files except .git are retained, without filtering or spectool. Host +rpmautospec generate-changelog/calculate-release are used only when needed for +release history or provenance. Provenance is captured before carryover/bumping and inlined with %define. +Lockless output adds no macros file, load directive, or Source9999. Locked components keep their existing behavior. + +Local lockless components copy local files except the TOML defining the spec path. +Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. +Unchanged local non-autorelease components must manually update Release and changelog. +Local autorelease components can use --no-change-rebuild to record a timestamp. +Rendered output is automatically committed unless --no-git-commit is provided. +For local components, --changelog-comment supplies only the Git commit message; +without it, rendering collects the message in an editor before replacing output. +--no-git-commit skips this prompt. Output must be inside the project repository. + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config @@ -30,6 +69,13 @@ preserved. When using a custom output directory (--output-dir), --force is required alongside --clean-stale as a safety measure. This flag is only valid with -a. +Before rendering any lockless component, check for uncommitted tracked changes +and untracked files outside 'project.rendered-specs-dir' and in each selected +component's dist-git directory. Other changes under the configured rendered-specs +directory are ignored. Git-ignored files are excluded. +--allow-uncommitted bypasses this check. +--check-only and --no-git-commit do not bypass it. Locked components are unaffected. + ``` azldev component render [flags] ``` @@ -45,15 +91,14 @@ azldev component render [flags] # Render to a custom directory, allowing removal of existing rendered component directories azldev component render -a -o rendered/ --force - - # Render all and remove stale directories - azldev component render -a --clean-stale ``` ### Options ``` -a, --all-components Include all components + --allow-uncommitted allow lockless rendering with uncommitted or untracked files in the project repository + --changelog-comment string Git commit message for lockless components and verbatim rpmdev-bumpspec -c comment for upstream non-autorelease specs (if omitted, prompt in an editor) --check-only render to a staging area and compare against the existing on-disk output without modifying the output directory. Exits 0 when nothing would change and 1 when any component would drift. With -a + --clean-stale, also fails on orphan rendered-spec directories. Intended for CI gates. --clean-stale prune rendered-spec directories that no longer correspond to a configured component (only with -a; requires -f with -o). Top-level non-component siblings are preserved. -p, --component stringArray Component name pattern @@ -61,6 +106,8 @@ azldev component render [flags] --fail-on-error exit with error if any component fails to render (useful for CI) -f, --force allow overwriting existing rendered component directories -h, --help help for render + --no-change-rebuild allow unchanged upstream renders and local autorelease renders; record a timestamp only for autorelease + --no-git-commit leave lockless rendered output uncommitted without staging it -o, --output-dir string output directory for rendered specs (overrides rendered-specs-dir from config) --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation --skip-lock-validation skip lock file consistency checks @@ -83,7 +130,6 @@ azldev component render [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_test.md b/docs/user/reference/cli/azldev_component_test.md index e316666e3..363cc8870 100644 --- a/docs/user/reference/cli/azldev_component_test.md +++ b/docs/user/reference/cli/azldev_component_test.md @@ -109,7 +109,6 @@ azldev component test COMPONENT [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_component_update.md b/docs/user/reference/cli/azldev_component_update.md index 024d3a153..acd88e290 100644 --- a/docs/user/reference/cli/azldev_component_update.md +++ b/docs/user/reference/cli/azldev_component_update.md @@ -14,6 +14,10 @@ For local components, this computes a content hash of the spec directory. Subsequent commands (render, build) use the locked state for deterministic, reproducible results. +Components listed in lockless-components.txt beside the top-level azldev.toml +are skipped, including with --bump. Use refresh-upstream-commit for them. +Their existing lock files are left untouched. + If a component fails to resolve, successful components are still written before the command returns an error. Failed components are left unchanged, and orphan pruning is skipped for that run. @@ -84,7 +88,6 @@ azldev component update [flags] -q, --quiet only enable minimal output --spec-editor editor Select the RPM spec editor (legacy or experimental) (default legacy) -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config.md b/docs/user/reference/cli/azldev_config.md index 6eed54b63..e62fe0781 100644 --- a/docs/user/reference/cli/azldev_config.md +++ b/docs/user/reference/cli/azldev_config.md @@ -32,7 +32,6 @@ JSON schema used for validating TOML config files. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config_dump.md b/docs/user/reference/cli/azldev_config_dump.md index 7136efc12..2f9860bc1 100644 --- a/docs/user/reference/cli/azldev_config_dump.md +++ b/docs/user/reference/cli/azldev_config_dump.md @@ -52,7 +52,6 @@ azldev config dump [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_config_generate-schema.md b/docs/user/reference/cli/azldev_config_generate-schema.md index 2f4eece5a..d92980891 100644 --- a/docs/user/reference/cli/azldev_config_generate-schema.md +++ b/docs/user/reference/cli/azldev_config_generate-schema.md @@ -46,7 +46,6 @@ azldev config generate-schema [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs.md b/docs/user/reference/cli/azldev_docs.md index 787033cbb..251289417 100644 --- a/docs/user/reference/cli/azldev_docs.md +++ b/docs/user/reference/cli/azldev_docs.md @@ -32,7 +32,6 @@ command tree, suitable for inclusion in the user guide. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent.md b/docs/user/reference/cli/azldev_docs_agent.md index fd1c785b0..e72765620 100644 --- a/docs/user/reference/cli/azldev_docs_agent.md +++ b/docs/user/reference/cli/azldev_docs_agent.md @@ -34,7 +34,6 @@ reference so that agents always load the guidance that ships with the binary. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent_install.md b/docs/user/reference/cli/azldev_docs_agent_install.md index c8c8853a5..638938eef 100644 --- a/docs/user/reference/cli/azldev_docs_agent_install.md +++ b/docs/user/reference/cli/azldev_docs_agent_install.md @@ -31,8 +31,8 @@ MCP server. Directory paths in the emitted content (such as the lock, generated upstream-commit, and rendered-spec directories) are resolved from the loaded azldev.toml, falling back to azldev's built-in defaults when no configuration is -found. The emitted content also reflects the mode azldev runs in, so pass ---without-lockfile to describe the lock-file-free workflow. The bindings reflect +found. By default, the content includes both lock-file and lockless workflows +selected per component by lockless-components.txt. The bindings reflect the project azldev runs in, so pair --output-dir with -C pointing at the target repository when scaffolding a different repo. @@ -74,7 +74,6 @@ azldev docs agent install [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_agent_show.md b/docs/user/reference/cli/azldev_docs_agent_show.md index 11d32383c..18a5dad11 100644 --- a/docs/user/reference/cli/azldev_docs_agent_show.md +++ b/docs/user/reference/cli/azldev_docs_agent_show.md @@ -51,7 +51,6 @@ azldev docs agent show [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_docs_markdown.md b/docs/user/reference/cli/azldev_docs_markdown.md index a86b59b4d..4e70f63bc 100644 --- a/docs/user/reference/cli/azldev_docs_markdown.md +++ b/docs/user/reference/cli/azldev_docs_markdown.md @@ -53,7 +53,6 @@ azldev docs markdown [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image.md b/docs/user/reference/cli/azldev_image.md index ea678cf94..75579f3ca 100644 --- a/docs/user/reference/cli/azldev_image.md +++ b/docs/user/reference/cli/azldev_image.md @@ -33,7 +33,6 @@ can be customized using Azure Linux Image Customizer. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_boot.md b/docs/user/reference/cli/azldev_image_boot.md index 143536805..24ccc97bc 100644 --- a/docs/user/reference/cli/azldev_image_boot.md +++ b/docs/user/reference/cli/azldev_image_boot.md @@ -97,7 +97,6 @@ azldev image boot [IMAGE_NAME] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_build.md b/docs/user/reference/cli/azldev_image_build.md index 3c89f6b7a..d2aa0c8bf 100644 --- a/docs/user/reference/cli/azldev_image_build.md +++ b/docs/user/reference/cli/azldev_image_build.md @@ -58,7 +58,6 @@ azldev image build [image-name] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_customize.md b/docs/user/reference/cli/azldev_image_customize.md index 929c4f10b..a1125258d 100644 --- a/docs/user/reference/cli/azldev_image_customize.md +++ b/docs/user/reference/cli/azldev_image_customize.md @@ -56,7 +56,6 @@ azldev image customize [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_inject-files.md b/docs/user/reference/cli/azldev_image_inject-files.md index 748bde7f4..7dcd71ebb 100644 --- a/docs/user/reference/cli/azldev_image_inject-files.md +++ b/docs/user/reference/cli/azldev_image_inject-files.md @@ -47,7 +47,6 @@ azldev image inject-files [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_list.md b/docs/user/reference/cli/azldev_image_list.md index a1270bbd3..d22c8b4a4 100644 --- a/docs/user/reference/cli/azldev_image_list.md +++ b/docs/user/reference/cli/azldev_image_list.md @@ -49,7 +49,6 @@ azldev image list [image-name-pattern...] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index e2c1ca54e..cb5e9bdbe 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -82,7 +82,6 @@ azldev image test IMAGE_NAME [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_package.md b/docs/user/reference/cli/azldev_package.md index 678421bcc..fc4e5a176 100644 --- a/docs/user/reference/cli/azldev_package.md +++ b/docs/user/reference/cli/azldev_package.md @@ -33,7 +33,6 @@ publish channel assignments derived from package groups and component overrides. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_package_list.md b/docs/user/reference/cli/azldev_package_list.md index e6165273e..39ce75870 100644 --- a/docs/user/reference/cli/azldev_package_list.md +++ b/docs/user/reference/cli/azldev_package_list.md @@ -82,7 +82,6 @@ azldev package list [package-name...] [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project.md b/docs/user/reference/cli/azldev_project.md index fef53ace7..ea647dae6 100644 --- a/docs/user/reference/cli/azldev_project.md +++ b/docs/user/reference/cli/azldev_project.md @@ -32,7 +32,6 @@ as an Azure Linux project with a basic configuration. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project_init.md b/docs/user/reference/cli/azldev_project_init.md index 13be79456..49de788c7 100644 --- a/docs/user/reference/cli/azldev_project_init.md +++ b/docs/user/reference/cli/azldev_project_init.md @@ -43,7 +43,6 @@ azldev project init [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_project_new.md b/docs/user/reference/cli/azldev_project_new.md index b494e46ee..e87182de9 100644 --- a/docs/user/reference/cli/azldev_project_new.md +++ b/docs/user/reference/cli/azldev_project_new.md @@ -46,7 +46,6 @@ azldev project new PATH [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo.md b/docs/user/reference/cli/azldev_repo.md index ccbc3f5ee..7d4091164 100644 --- a/docs/user/reference/cli/azldev_repo.md +++ b/docs/user/reference/cli/azldev_repo.md @@ -33,7 +33,6 @@ under one or more URL prefixes. -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo_compare.md b/docs/user/reference/cli/azldev_repo_compare.md index 7053a3f19..dcb59b9b6 100644 --- a/docs/user/reference/cli/azldev_repo_compare.md +++ b/docs/user/reference/cli/azldev_repo_compare.md @@ -51,7 +51,6 @@ azldev repo compare [flags] -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_repo_query.md b/docs/user/reference/cli/azldev_repo_query.md index 9086aa1c9..f0c5cc540 100644 --- a/docs/user/reference/cli/azldev_repo_query.md +++ b/docs/user/reference/cli/azldev_repo_query.md @@ -74,7 +74,6 @@ azldev repo query [flags] -- -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/cli/azldev_version.md b/docs/user/reference/cli/azldev_version.md index 5ebddbcdd..27febae21 100644 --- a/docs/user/reference/cli/azldev_version.md +++ b/docs/user/reference/cli/azldev_version.md @@ -37,7 +37,6 @@ azldev version -O json -C, --project string path to Azure Linux project -q, --quiet only enable minimal output -v, --verbose enable verbose output - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files ``` ### SEE ALSO diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index ce06e6f60..5e1324622 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -76,6 +76,11 @@ When the upstream package has a different name than the component, use `upstream spec = { type = "upstream", upstream-name = "redhat-rpm-config" } ``` +Upstream clones fetch only the effective distro version's `dist-git-branch`, +including its full history. They are not shallow clones and do not fetch every +upstream branch. Select a different branch through `spec.upstream-distro` and +the referenced distro version's `dist-git-branch`. + ### Commit-Pinned Specs When you need to pin to a specific git commit — for example, to pick up a fix that hasn't been included in a tagged release yet — use `upstream-commit`: @@ -87,6 +92,10 @@ spec = { type = "upstream", upstream-commit = "a1b2c3d4e5f6789" } The value must be a hex string between 7 and 40 characters (a short or full git commit hash). When `upstream-commit` is set, it takes priority over the distro snapshot date — azldev checks out the exact commit instead of finding the latest commit before a snapshot timestamp. +The pinned commit must be available from the selected upstream branch. Lockless +rendering additionally fetches the last render's branch when it differs and +the old and current pins must be compared. + > **Note:** Commit-pinning is intended for temporary use. Once the desired change lands in a tagged upstream release, switch back to version-based pinning or the default snapshot to keep the component aligned with the upstream distro. ### Local Specs @@ -131,7 +140,7 @@ calculation = "manual" ### Optional `rpmdev-bumpspec` preview -By default, build, render, and `prepare-sources` use the legacy static Release +For lock-file-based components, build, render, and `prepare-sources` use the legacy static Release calculation above. It supports only a bare integer or an integer followed by `%{?dist}` or `%{dist}`. Other static forms, such as dotted releases or custom macros, retain the existing non-standard Release error and require `manual` or @@ -142,19 +151,35 @@ or `azldev component prepare-sources` to preview transactional `rpmdev-bumpspec` handling instead. This temporary Azure Linux migration flag uses the existing fixed changelog metadata and requires `rpmdev-bumpspec`, `rpmdev-packager`, `rpm`, `rpmspec`, and Python with the RPM module only when -explicitly selected. `%autorelease` behavior is unchanged. +explicitly selected for this transactional strategy. `%autorelease` behavior is unchanged. + +**Exception:** Rendering a lockless upstream component with a non-autorelease +Release invokes host `rpmdev-bumpspec -c` without requiring the flag. +Autorelease renders skip bumpspec. See the +[lockless rendering workflow](../../how-to/preview-without-lockfile.md#render-and-commit-an-upstream-component) +for release/changelog carryover and automatic output commits. + +Lockless builds instead use committed rendered dist-git history and Mock's +rpmautospec plugin; they reject `--rpmdev-bumpspec` and `--without-git`. +Lockless `prepare-sources` is unsupported; use `component dist-git-repo` instead. ## Render Configuration The `[components..render]` section controls rendering behavior for a component. +Lock mode for every command is selected by `lockless-components.txt` beside the +top-level `azldev.toml`: listed components use the lockless workflow, and +unlisted components use lock files. A missing or empty list selects lock-file +behavior for every component. List every component for a fully lockless project. See +[per-component lock modes](../../how-to/preview-without-lockfile.md#select-lock-mode-per-component). + | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| -| Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering (defaults to `false`) | +| Skip file filter | `skip-file-filter` | boolean | No | Disable post-render file filtering for locked components (defaults to `false`; ignored for lockless components) | ### Skip File Filter -During rendering, azldev uses `spectool` to determine which files are referenced by `Source` and `Patch` tags in the spec, then removes unreferenced files from the rendered output. Some specs use dynamic macros (e.g., `%{fontpkgname1}`) that `spectool` cannot expand, causing it to report incorrect filenames. This results in referenced files being incorrectly removed. +When rendering locked components, azldev uses `spectool` to determine which files are referenced by `Source` and `Patch` tags in the spec, then removes unreferenced files from the rendered output. Some specs use dynamic macros (e.g., `%{fontpkgname1}`) that `spectool` cannot expand, causing it to report incorrect filenames. This results in referenced files being incorrectly removed. Set `skip-file-filter = true` to preserve all files from the dist-git checkout: @@ -163,7 +188,11 @@ Set `skip-file-filter = true` to preserve all files from the dist-git checkout: skip-file-filter = true ``` -> **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most components, the default filtering behavior is correct and keeps the rendered output clean. +> **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most locked components, the default filtering behavior is correct and keeps the rendered output clean. + +Lockless rendering does not query `spectool` or filter files: it always retains +all prepared dist-git files except `.git`. This setting has no effect on +components selected by `lockless-components.txt`. ## Build Configuration @@ -228,9 +257,13 @@ emit-upstream-provenance = true | `%fedora_upstream_version` | The `Version` tag from the pristine upstream Fedora spec | | `%fedora_upstream_release` | The `Release` tag from the pristine upstream Fedora spec, with `%{?dist}` expanded to the Fedora dist tag (e.g. `.fc43`) | -The values are read from the upstream spec **before** any azldev overlays are applied, so they reflect the true upstream Name-Version-Release, not the Azure Linux–modified spec. The macros are derived fresh at render/build time from the pinned upstream commit and emitted into the component's generated macros file (loaded via `%{load:...}`). +The values are captured immediately after fetching the pinned upstream checkout, +**before** release/changelog carryover, release bumping, or overlays can modify +the spec. The captured values are reused when emitting inline `%define` entries +for lockless components, or writing the generated macros file (loaded via +`%{load:...}`) for locked components, rather than querying the modified checkout. -For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` in the project distro's mock chroot against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). This keeps `rpmautospec` out of azldev's host dependencies. If the project distro has no mock config, or mock resolution fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`. +For specs whose `Release` uses rpmautospec (`Release: %autorelease`), the pristine Fedora release number is computed by running `rpmautospec calculate-release` against the upstream dist-git checkout (whose history is Fedora's, not azldev's synthetic overlay history). Lockless rendering runs this query on the host; other workflows use the project distro's mock chroot. The query runs only when overlays will be applied, provenance is enabled for a Fedora upstream component, and `build.defines` does not already supply `fedora_upstream_release`. Static releases do not need this query. If resolution is unavailable or fails, `%fedora_upstream_release` is skipped (with a warning) rather than emitting the literal `%autorelease`; the query is not retried when emitting macros. Example: for a component pinned to Fedora 43's `grub2-2.12-5.fc43`, the spec can reference: @@ -244,7 +277,7 @@ Notes: - The flag has no effect on local or SRPM components (they have no upstream provenance) or on non-Fedora upstreams; for those it is silently ignored. - If a component defines a macro of the same name via `build.defines`, the user-defined value wins — the injected value does not overwrite it. -- **How tag values are resolved:** `Version` and `Release` are read as plain text from the pristine upstream spec. `%{?dist}` is substituted with the Fedora dist tag, and `%autorelease` is resolved to a concrete number via `rpmautospec` (see above). Any *other* in-spec macros — e.g. `Version: %{majorver}.%{minorver}` — are emitted into the macros file unexpanded. Because that file is loaded back into the same spec via `%{load:...}`, RPM expands them lazily at build time using the spec's own macro definitions, so the consuming spec still sees the correct fully-expanded value. +- **How tag values are resolved:** `Version` and `Release` are read as plain text from the pristine upstream spec. `%{?dist}` is substituted with the Fedora dist tag, and `%autorelease` is resolved to a concrete number via `rpmautospec` (see above). Any *other* in-spec macros — e.g. `Version: %{majorver}.%{minorver}` — remain unexpanded in the generated definitions. Lockless specs use lazy `%define` entries; locked specs load their macros file via `%{load:...}`. Both expand these references at build time using the spec's own macro definitions, so the consuming spec still sees the correct fully-expanded value. Limitations: diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index 258e9cf2b..ffc719e09 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -39,7 +39,7 @@ Glob patterns that match no files are silently ignored. Literal filenames (no wi Includes are resolved recursively — included files can themselves declare further includes. For a detailed explanation of load order and merge semantics, see [Configuration System](../../explanation/config-system.md). -> **Preview:** With the global `--without-lockfile` flag, the generated upstream-commit files must be included before any component-specific TOML configuration, so that later component definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). +> **Preview:** For components listed in `lockless-components.txt` beside the top-level `azldev.toml`, include generated upstream-commit files before component-specific TOML configuration so later definitions supply the remaining `spec` fields and may explicitly override a generated pin. See [Preview the Lock-File-Free Mode](../../how-to/preview-without-lockfile.md). ## Minimal Example diff --git a/internal/app/azldev/agentskill/agentskill.go b/internal/app/azldev/agentskill/agentskill.go index f700a8712..6d43365b4 100644 --- a/internal/app/azldev/agentskill/agentskill.go +++ b/internal/app/azldev/agentskill/agentskill.go @@ -211,8 +211,8 @@ type Catalog struct { withoutLockfile bool } -// NewCatalog returns the catalog for the selected mode. Pass true to describe -// lock-file-free mode, as selected by the global '--without-lockfile' flag. +// NewCatalog returns the default mixed-workflow catalog, or an internal +// all-lockless variant when withoutLockfile is true. The CLI uses the default. func NewCatalog(withoutLockfile bool) Catalog { return Catalog{withoutLockfile: withoutLockfile} } @@ -231,7 +231,7 @@ func (c Catalog) Skills() []Skill { result := slices.Clone(skills) if !c.withoutLockfile { - return result + return append(result, withoutLockfileSkills[updateComponentSkillName]) } for idx := range result { diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index a263a100f..fe912b439 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -160,6 +160,169 @@ func TestComponentTomlSkillDocumentsRPMDevBumpspecPreview(t *testing.T) { assert.Contains(t, doc, "no bumpspec runtime discovery occurs") } +func TestComponentTomlSkillDocumentsPerComponentRenderModes(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev-comp-toml", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "`lockless-components.txt` beside the top-level `azldev.toml`") + assert.Contains(t, doc, "all others use lock files") + assert.Contains(t, doc, "for every command") + assert.Contains(t, doc, "two blank lines and `#`-commented instructions") + assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") + assert.Contains(t, doc, "Local lockless components skip upstream carryover; Release and changelog remain unchanged") + assert.Contains(t, doc, "TOML defining the local spec path") + assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") + assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "both stdin and stdout to be TTYs") + assert.Contains(t, doc, "Autorelease with `--no-git-commit` never prompts") + assert.Contains(t, doc, "Blank/comment-only edited messages fail rendering") + assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") + } +} + +func TestSkillsDocumentDistGitRepo(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "azldev comp dist-git-repo -o ") + assert.Contains(t, doc, "--single-branch --no-local") + assert.Contains(t, doc, "specs//") + assert.Contains(t, doc, "and git-filter-repo") + } +} + +func TestSkillsDocumentLocklessRPMContext(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "temporary process-local RPM wrapper sets `_sourcedir` and `_specdir`") + assert.Contains(t, doc, "without changing `HOME` or editing the user's `.rpmmacros`") + assert.Contains(t, doc, "removed after the invocation") + } +} + +func TestSkillsDocumentLocklessBuildAndQuery(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + catalog := agentskill.NewCatalog(withoutLockfile) + doc, err := catalog.SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "`comp query` extracts committed rendered history") + assert.Contains(t, doc, "Lockless `comp prepare-sources` writes directly to `--output-dir`") + assert.Contains(t, doc, "including modified and untracked files.") + assert.Contains(t, doc, "committed component dist-git files; edits in the existing dist-git directory are") + assert.Contains(t, doc, "removes `.git` after successful preparation, preserving it on failure.") + assert.Contains(t, doc, "It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`.") + assert.Contains(t, doc, "from `--output-dir`") + assert.Contains(t, doc, "locked queries still read the") + doc, err = catalog.SkillDocument("azldev-build-component", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "Mock's rpmautospec plugin (Mock 5.3+)") + assert.Contains(t, doc, "Uncommitted rendered edits are ignored") + assert.Contains(t, doc, "Locked builds retain their existing") + } +} + +func TestSkillsDocumentAutomaticLocklessRenderCommits(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "host `rpmdev-bumpspec`") + assert.Contains(t, doc, "`--check-only` never creates project commits") + assert.Contains(t, doc, "never uses mock or `rpmautospec process-distgit`") + assert.Contains(t, doc, "all prepared dist-git files except `.git`") + assert.NotContains(t, doc, "host `spectool` filtering") + assert.Contains(t, doc, "prepares the changelog before Release") + assert.Contains(t, doc, "Git-ignored files are excluded") + assert.Contains(t, doc, "and in selected components' dist-git directories. Other rendered changes are ignored.") + assert.Contains(t, doc, "`--allow-uncommitted`") + assert.Contains(t, doc, "first commit also includes") + assert.Contains(t, doc, "Other rendered paths are excluded even if staged") + assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") + assert.Contains(t, doc, "Overlays can override the prepared release and changelog") + assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") + assert.Contains(t, doc, "Unchanged lockless renders require `--no-change-rebuild`") + assert.Contains(t, doc, "Local lockless components skip upstream carryover; Release and changelog remain unchanged") + assert.Contains(t, doc, "TOML defining the local spec path") + assert.Contains(t, doc, "participate in unchanged-render checks and automatic commits") + assert.Contains(t, doc, "host `date -Is` output") + assert.Contains(t, doc, "Upstream non-autorelease rebuilds proceed without a timestamp marker") + assert.Contains(t, doc, "must manually update their release and changelog") + assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") + assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") + assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") + assert.Contains(t, doc, "The new NEVR must be equal or higher for a final autorelease spec") + assert.Contains(t, doc, "non-autorelease requires a higher NEVR") + assert.Contains(t, doc, "`--changelog-comment`") + assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") + assert.Contains(t, doc, "`$EDITOR` or a local editor") + assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "Local components with `--no-git-commit` never prompt") + assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") + assert.Contains(t, doc, "two blank lines and `#`-commented instructions") + assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") + assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") + assert.Contains(t, doc, "`azldev comp make-lockless`") + assert.Contains(t, doc, "lexical order") + assert.Contains(t, doc, "without blank lines or duplicates") + assert.Contains(t, doc, "spaces and tabs after `Release:`") + } + + doc, err := agentskill.SkillDocument("azldev-refresh-upstream-commit", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "fully parsed committed TOML") + assert.Contains(t, doc, "Git-ignored files are excluded") + assert.Contains(t, doc, "and in selected components' dist-git directories. Other rendered changes are ignored.") + assert.Contains(t, doc, "`--allow-uncommitted`") + assert.Contains(t, doc, "`render --allow-uncommitted`") + assert.Contains(t, doc, "automatically commits that directory") + assert.Contains(t, doc, "first commit also includes") + assert.Contains(t, doc, "Other rendered paths are excluded even if staged") + assert.Contains(t, doc, "## RPMAUTOSPEC:") + assert.Contains(t, doc, "leading integer of the committed,") + assert.Contains(t, doc, "an omitted `-b` stays omitted") + assert.Contains(t, doc, "committed file byte-for-byte") + assert.Contains(t, doc, "Static release carryover runs only when committed output exists and the version") + assert.Contains(t, doc, "Initial and legacy autorelease handling do not depend on version changes") + assert.Contains(t, doc, "base only when the version is unchanged") + assert.Contains(t, doc, "When the version changes, rendering calculates the upstream numeric base and sets `-b`") + assert.Contains(t, doc, "The committed changelog is still carried over, not regenerated") + assert.Contains(t, doc, "A calculated base of `1` omits `-b`") + assert.Contains(t, doc, "Changelog handling precedes release preparation and is independent of version") + assert.Contains(t, doc, "retains `%autorelease`, `%autochangelog`") + assert.Contains(t, doc, "effective upstream distro's `dist-git-branch`") + assert.Contains(t, doc, "fetches that historical branch before comparing commits") + assert.Contains(t, doc, "fetches that specific commit hash") + assert.Contains(t, doc, "update to upstream commit ") + assert.Contains(t, doc, "All prepared dist-git files except `.git` are retained") + assert.Contains(t, doc, "immediately after checking out the pinned upstream commit") + assert.Contains(t, doc, "that step never repeats the provenance query") + assert.Contains(t, doc, "the changelog before Release, then emits inline macros") + assert.Contains(t, doc, "Version detection and carryover use the upstream spec before overlays") + assert.Contains(t, doc, "Overlays can override the prepared release and changelog") + assert.Contains(t, doc, "compares output before the final release bump; `--check-only` stops before that bump") + assert.Contains(t, doc, "Unchanged lockless renders require `--no-change-rebuild`") + assert.Contains(t, doc, "host `date -Is` output") + assert.Contains(t, doc, "Upstream non-autorelease rebuilds proceed without a timestamp marker") + assert.Contains(t, doc, "must manually update their release and changelog") + assert.Contains(t, doc, "ignores this root marker in both existing output and the working upstream checkout") + assert.Contains(t, doc, "Lockless failures never write `RENDER_FAILED`") + assert.Contains(t, doc, "Recovery discards uncommitted changes in that directory and keeps no backups") + assert.Contains(t, doc, "The new NEVR must be equal or higher for a final autorelease spec") + assert.Contains(t, doc, "non-autorelease requires a higher NEVR") + assert.Contains(t, doc, "`--changelog-comment`") + assert.Contains(t, doc, "Autorelease skips `rpmdev-bumpspec`") + assert.Contains(t, doc, "`$EDITOR` or a local editor") + assert.Contains(t, doc, "The final Git commit never prompts") + assert.Contains(t, doc, "Autorelease with `--no-git-commit` never prompts") + assert.Contains(t, doc, "Each change line is prefixed with `- ` and is editable") + assert.Contains(t, doc, "two blank lines and `#`-commented instructions") + assert.Contains(t, doc, "Blank lines and lines starting with `#` are removed, with no trailing newline") + assert.Contains(t, doc, "`--no-git-commit` skips automatic staging and committing") + assert.Contains(t, doc, "`azldev comp make-lockless`") + assert.Contains(t, doc, "spaces and tabs after `Release:`") + assert.NotContains(t, doc, "git commit --amend") +} + func TestImageSkillDocumentsSKUGroupGuidance(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-image", testParams()) require.NoError(t, err) @@ -548,18 +711,18 @@ func TestImageSkillCoversCapabilities(t *testing.T) { } // TestCatalog_SkillsByMode verifies that each mode advertises the skill describing -// how it maintains resolved component state, and only that one. +// how it maintains resolved component state, including both workflows by default. func TestCatalog_SkillsByMode(t *testing.T) { defaultNames := skillNamesOf(agentskill.NewCatalog(false)) assert.Contains(t, defaultNames, "azldev-update-component") - assert.NotContains(t, defaultNames, "azldev-refresh-upstream-commit") + assert.Contains(t, defaultNames, "azldev-refresh-upstream-commit") withoutLockfileNames := skillNamesOf(agentskill.NewCatalog(true)) assert.Contains(t, withoutLockfileNames, "azldev-refresh-upstream-commit") assert.NotContains(t, withoutLockfileNames, "azldev-update-component") // The two catalogs otherwise describe the same set of skills, in the same order. - assert.Len(t, withoutLockfileNames, len(defaultNames)) + assert.Len(t, withoutLockfileNames, len(defaultNames)-1) } // TestCatalog_SkillContentByMode verifies that the rendered skill bodies describe diff --git a/internal/app/azldev/agentskill/content/azldev.md.tmpl b/internal/app/azldev/agentskill/content/azldev.md.tmpl index f3a572f86..db0cae5b9 100644 --- a/internal/app/azldev/agentskill/content/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/azldev.md.tmpl @@ -34,6 +34,83 @@ images assemble RPMs into bootable artifacts. ## Common commands +`lockless-components.txt` beside the top-level `azldev.toml` selects the lockless +workflow per component for all commands. Listed components ignore locks and use +generated upstream-commit TOML instead. All others use locks. +`azldev comp make-lockless` adds selected exact names to that list if needed, +then runs the refresh pipeline. The list is written in lexical order, one name +per line without blank lines or duplicates. Existing names and lock files are retained. +Already-listed components are still refreshed; a refresh failure leaves the list updated. + +Before preparing any selected lockless component, rendering checks staged changes, +unstaged tracked changes, and untracked files outside `project.rendered-specs-dir` +and in selected components' dist-git directories. Other rendered changes are ignored. +Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty +renders; `--check-only` and `--no-git-commit` do not bypass this check. +The override does not change committed-history carryover or failure recovery. +Only locked components do not trigger this check. + +For listed upstream components, rendering prepares the changelog before Release, +then writes macros, applies overlays, and updates `sources`. +Version detection and carryover use the upstream spec before overlays. +Overlays can override the prepared release and changelog. +Non-autorelease rendering then runs host `rpmdev-bumpspec`, without synthetic history or its legacy +release adjustment. A temporary process-local RPM wrapper sets `_sourcedir` and `_specdir` +to the spec directory without changing `HOME` or editing the user's `.rpmmacros`; +it is removed after the invocation, including on failure. Lockless rendering +preserves the existing spaces and tabs after `Release:` when editing its value, and +compares output before the final release bump; `--check-only` stops before that bump. +Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors +without replacing existing output. For unchanged autorelease components (upstream or local), +the flag records host `date -Is` output in `.no-change-rebuild`. +Upstream non-autorelease rebuilds proceed without a timestamp marker. +Unchanged local non-autorelease components must manually update their release and changelog, +even with `--no-change-rebuild`. Comparison +ignores this root marker in both existing output and the working upstream checkout. +Lockless failures never write `RENDER_FAILED`. Before replacement, output is +untouched; failures after replacement starts clear the component directory and +use Git to restore committed output, or leave no directory if none existed. +Recovery discards uncommitted changes in that directory and keeps no backups. +Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. +The package name must match. The new NEVR must be equal or higher for a final autorelease spec; +non-autorelease requires a higher NEVR. Invalid values fail with both NEVRs shown. +Validation errors leave existing output untouched and fail even without `--fail-on-error`. +First renders, `--check-only`, and dry runs skip this replacement-time check. +`--no-change-rebuild` never bypasses the NEVR check. +It never uses mock or `rpmautospec process-distgit`; RPM macros remain unexpanded. +It retains all prepared dist-git files except `.git`, without file filtering or +`spectool` queries, and replaces and +automatically commits that component's output directory. The first commit also includes +all staged changes, unstaged tracked changes, and non-ignored untracked files outside +the rendered output directory. Other rendered paths are excluded even if staged; +later commits include only their component's dist-git changes. +`--changelog-comment` supplies the exact `rpmdev-bumpspec -c` +value and commit message, with no default. If omitted, `$EDITOR` or a local editor +collects the entry using a template with two blank lines and `#`-commented instructions +when no upstream changes exist. Otherwise, line 1 is blank for the summary; +non-autorelease changes start on line 2, while autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. +Autorelease skips `rpmdev-bumpspec`; missing messages are collected before output replacement. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Provide `--changelog-comment` for unattended rendering. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +Upstream non-autorelease still needs `--changelog-comment` even with `--no-git-commit`. +`--no-git-commit` skips automatic staging and committing, but still renders and +bumps non-autorelease specs. +Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. +Commit refreshed TOML before rendering. Configure Git author +identity and install `rpmdevtools`. `--check-only` never creates project commits. +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, while retaining other TOML sidecars. +They participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message; otherwise +rendering collects it before output replacement, with commit-only instructions for non-autorelease. +Local components with `--no-git-commit` never prompt. Locked components retain their existing render behavior. + Top-level commands in this build: {{ range .TopLevelCommands }}- `azldev {{ .Name }}` — {{ .Short }} @@ -47,12 +124,45 @@ Everyday tasks (add `-q -O json` when scripting): | Add a component | `azldev comp add` | | Build a component | `azldev comp build -p ` | | Render specs | `azldev comp render -p ` (or `-a`) | +| Extract committed dist-git history | `azldev comp dist-git-repo -o ` | | Refresh a lock | `azldev comp update -p ` | +| Refresh a lockless component | `azldev comp refresh-upstream-commit -p ` | | List / build images | `azldev image list` / `azldev image build` | `comp` is an alias for `component`. Always confirm current syntax with `azldev --help`. +`comp dist-git-repo` requires exactly one component name, `--output-dir` (`-o`), +and git-filter-repo. It clones the local project's current branch with +`--single-branch --no-local`, then filters +`{{ .RenderedSpecsDir }}//` into the repository root. +Only committed rendered files/history are exported; it does not fetch upstream. +Existing output requires `--force`, which deletes it after preflight checks. + +For lockless components, `comp query` extracts committed rendered history into +temporary dist-git repositories and processes it with rpmautospec in Mock before +querying. Uncommitted rendered edits are ignored; locked queries still read the +working tree. Lockless `comp prepare-sources` writes directly to `--output-dir` +using render-style preparation. Current TOML and file inputs use the working tree, +including modified and untracked files. Release/changelog baselines use only +committed component dist-git files; edits in the existing dist-git directory are +ignored. Nonempty output requires `--force`; failures can +leave partial output. It downloads/generates sources before inline macro setup and +overlays. Lockless specs use `%define` entries, with no separate macros file, +load directive, or generated `Source9999`; locked specs retain their macros file. +Archive overlays run before spec and +loose-file overlays. It does not compare +rendered output, apply a final render release bump, or commit. It reads render history from +`project.rendered-specs-dir` using `/`, never +from `--output-dir`. It never generates synthetic history. `--without-git` +removes `.git` after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Skipping overlays also skips inline macro setup and `sources` metadata updates; +skipping sources also skips archive overlays. Missing hashes cannot be computed +when source acquisition is skipped. +Use `comp dist-git-repo` for committed history extraction. +`comp history` continues to skip lockless components. + The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. @@ -65,6 +175,7 @@ mock helpers; it is intentionally omitted from normal help. - Build, iterate, and debug a component — read the `azldev-build-component` skill. - Build, boot, and register images — read the `azldev-image` skill. - Refresh a lock and finalize for a PR — read the `azldev-update-component` skill. +- Refresh a lockless component — read the `azldev-refresh-upstream-commit` skill. - Remove a component — read the `azldev-remove-component` skill. - Test built RPMs in a chroot — read the `azldev-mock` skill. @@ -72,7 +183,7 @@ mock helpers; it is intentionally omitted from normal help. - **Never edit generated output** — rendered specs and the output/work/log dirs are produced by azldev. Change the source config and re-render. -- **Re-run `azldev comp update` before opening a PR** — the lock fingerprint covers +- **For locked components, re-run `azldev comp update` before opening a PR** — the lock fingerprint covers the whole component config, and lock / rendered-spec CI checks run on committed state. - **Every overlay needs a `description`** explaining why the change is needed. diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 9375d5e22..4882f78ff 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -23,6 +23,26 @@ progress — use it only for inner-loop builds you expect to succeed. ## The inner loop +For lockless components, build uses committed rendered dist-git history extracted +with git-filter-repo; render and commit first. It fetches configured sources through +the source manager (including download/custom origins), then remaining lookaside payloads, and +builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. +Build macros are inline `%define` entries in the committed spec, not passed again +as Mock command-line overrides. Lockless rendering adds no macros file, load +directive, or `Source9999` entry. Locked components retain their macros file. +Render and commit macro changes before building. +Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` +are rejected for lockless builds; `--no-check` is forwarded normally. +Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates +sources before inline macro setup, then applies archive overlays before spec and +loose-file overlays. History comes from `project.rendered-specs-dir`, not +`--output-dir`; no synthetic history is generated. `--without-git` removes `.git` +after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Nonempty output requires `--force`; failures can leave partial output. Use `comp dist-git-repo` +for committed history extraction. +Locked builds retain their existing source preparation and flag behavior. + investigate → modify → render → build → test → inspect | Step | Command | @@ -38,7 +58,14 @@ Prefer `comp render` for quick overlay verification; use `comp diff-sources` to exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the two trees). Builds can be slow — set generous timeouts. -Finalize with `azldev comp update -p ` before opening a PR (see the +For all lockless components, uncommitted tracked changes or untracked files outside +`project.rendered-specs-dir` or in selected components' dist-git directories require +`comp render --allow-uncommitted`. Other rendered changes are ignored. +Git-ignored files are excluded; the override does not change committed-history +carryover or failure recovery. Commit inputs before the final render (see the +`azldev-refresh-upstream-commit` skill). + +For locked components, finalize with `azldev comp update -p ` before opening a PR (see the `azldev-update-component` skill). ## Debugging build failures diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 0def4bf59..214cd0077 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -31,7 +31,7 @@ Add sub-tables only for what you change. The fields you will reach for most: | `overlays` / `overlay-files` | targeted spec/source edits (see the `azldev-overlays` skill) | | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | -| `render.skip-file-filter` | rendering edge-case escape hatch | +| `render.skip-file-filter` | locked-rendering edge-case escape hatch | ## Spec source @@ -86,19 +86,61 @@ calculation = "manual" ``` The temporary `--rpmdev-bumpspec` flag on `comp build`, `comp render`, and -`comp prepare-sources` opts into transactional host handling while Azure Linux +`comp prepare-sources` opts into transactional host handling for locked components while Azure Linux migrates. Only that path requires `rpmdev-bumpspec`, `rpmdev-packager`, `rpm`, `rpmspec`, and Python with its RPM module. It preserves the fixed team identity/comment/datestamp and strict source-EVR transaction checks; locks and synthetic history still determine the number and order of bumps. Without the flag, no bumpspec runtime discovery occurs and the legacy static calculation is -used. `%autorelease` behavior is unchanged on either path. +used for locked components. Lockless builds reject this flag and use Mock's +rpmautospec plugin with committed dist-git history; lockless `comp prepare-sources` +writes directly to `--output-dir` and downloads/generates sources before inline macro +setup, then applies archive overlays before spec/loose-file overlays. Lockless specs +use `%define` entries without a macros file, load directive, or generated `Source9999`. +Locked specs retain their separate macros file. Nonempty output +requires `--force`; failures can leave partial output. It does not commit changes. +It reads history from `project.rendered-specs-dir`, not +`--output-dir`, without generating synthetic history. `--without-git` removes `.git` +after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +The lockless upstream render workflow always runs +host `rpmdev-bumpspec -c` only for non-autorelease specs and automatically commits +the component output. Missing `--changelog-comment` opens `$EDITOR` or a local +editor before output replacement, including for autorelease commits. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. +Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. +Lockless rendering skips +synthetic history and its legacy release adjustment. Commit input TOML first. +The first automatic commit also includes pending repository changes outside the +rendered output directory; later commits contain only their component output. ## Render configuration -`render.skip-file-filter = true` keeps all source and patch files during render. -azldev normally prunes files not referenced by the rendered spec; set this only for +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, retain other TOML sidecars, +and participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message. +Local components with `--no-git-commit` never prompt. + +azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. +Listed component names use the lockless workflow for every command; +all others use lock files. A missing or empty list means all components use +locks. List every component to use a fully lockless workflow. +Use `comp refresh-upstream-commit` for listed components and `comp update` +for unlisted components; each command skips the other mode. + +`render.skip-file-filter = true` keeps all source and patch files during locked render. +Locked rendering normally prunes files not referenced by the rendered spec; set this only for the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. +Lockless rendering always retains all prepared dist-git files except `.git`, +without file filtering or `spectool` queries; this setting has no effect on it. ## File organization diff --git a/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl new file mode 100644 index 000000000..88d520014 --- /dev/null +++ b/internal/app/azldev/agentskill/content/refresh-upstream-commit.md.tmpl @@ -0,0 +1,208 @@ +--- +name: {{ .Name }} +description: {{ printf "%q" .Description }} +--- + +# Refresh component upstream commits + +`azldev comp refresh-upstream-commit` (`comp` is an alias for `component`) resolves one or more +components at the distro snapshot and writes normal component TOML under +`{{ .UpstreamCommitsDir }}/`. + +The project config must include `{{ .UpstreamCommitsDir }}/*.toml` **before** +the component-specific TOML configuration so subsequent commands load the +generated commit pins and the component definition supplies the remaining +`spec` fields. If `--upstream-commits-dir` selects another directory, change +the include pattern to match while preserving that ordering. + +The command uses the fully merged component configuration. It resolves and +writes a pin only when the selected component's effective `spec.type` is +`upstream`; for other source types, it removes any existing generated pin +without contacting an upstream provider. +It loads configuration permissively so a stale generated pin cannot block +cleanup after a component is removed or converted to another source type. +Other configuration validation failures are still reported as warnings. + +## When to run `refresh-upstream-commit` + +| Situation | Run `refresh-upstream-commit`? | +| --- | --- | +| Adding a new upstream component | **Yes** — first, to generate its upstream commit TOML | +| Finalizing overlay, build config, or metadata changes | No — these do not affect the resolved commit | +| Changing source resolution (commit pin, upstream distro/version, or snapshot) | **Yes** — also mid-workflow (see below) | +| Iterating on overlays / build config / metadata | No — use `render --allow-uncommitted` while inputs are uncommitted | +| Just reading or building existing components | No | + +Refresh a single component with `-p `. Use `-a` (all components) only for +coordinated mass refreshes (e.g. a new distro snapshot) or when investigating +commit drift across many components — it is slow. For day-to-day work use `-p`. +Add `-O json` for machine-readable output when debugging. + +## Finalizing component changes + +Select this workflow by listing the component's exact name in +`lockless-components.txt` beside the top-level `azldev.toml`. +Alternatively, `azldev comp make-lockless` adds selected exact names if needed +and runs this refresh pipeline. It accepts `-p`, `-g`, `-a`, or positional names, +writes the list in lexical order, one name per line without blank lines or duplicates, +preserves existing names and lock files, and refreshes already-listed components. +A refresh failure leaves the updated list in place; fix the failure and rerun. +Refresh skips unlisted components and preserves their generated TOML files. + +Commit component inputs before rendering. For upstream components, rendering +compares the fully parsed committed TOML at the last render commit and at HEAD; +uncommitted generated pins do not count as a committed upstream change. + +Before any upstream checkout or component preparation, rendering checks staged changes, +unstaged tracked changes, and untracked files outside `project.rendered-specs-dir` +and in selected components' dist-git directories. Other rendered changes are ignored. +Git-ignored files are excluded. Use `--allow-uncommitted` for intentional dirty +renders; `--check-only` and `--no-git-commit` do not bypass this check. +The override does not change committed-history carryover or failure recovery, +which can discard uncommitted files in the component output directory. +Only locked components do not trigger this check. +Local lockless components use the same scoped cleanliness check, exclude the TOML defining +the local spec path, and participate in unchanged-render checks and automatic commits. +Their Release and changelog remain unchanged; `--changelog-comment` supplies only +their Git message. + +```sh +git add \ + {{ .UpstreamCommitsDir }}/.toml +git commit -m "fix(): ..." +azldev comp render -p +``` + +Rendering clones only the effective upstream distro's `dist-git-branch`, keeping +its full history rather than making a shallow clone. If the last render's +committed TOML resolves to a different branch and the pins differ, rendering +fetches that historical branch before comparing commits, without switching the +working checkout. +If the old pin cannot be resolved, rendering fetches that specific commit hash +from `origin` and checks again. If it remains unavailable, rendering continues +with `update to upstream commit ` as the upstream +changes text, even when the historical branch fetch failed. Other Git log errors +remain fatal. + +With `build.emit-upstream-provenance`, rendering captures Fedora provenance +immediately after checking out the pinned upstream commit, before +release/changelog carryover or bumping. Host `rpmautospec calculate-release` +runs here only for an upstream `%autorelease` whose `fedora_upstream_release` +is not already supplied by `build.defines`. The captured values are reused +when emitting inline `%define` entries; that step never repeats the provenance query. + +After reading render history and collecting upstream changes, rendering prepares +the changelog before Release, then emits inline macros, applies overlays, and +updates `sources` before comparison and the final `rpmdev-bumpspec`. +Release edits preserve the existing spaces and tabs after `Release:`. +Version detection and carryover use the upstream spec before overlays. +Overlays can override the prepared release and changelog. +Carryover is not repeated after overlays, even if they change the version. +Rendering compares output before the final release bump; `--check-only` stops before that bump. +Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors +without replacing existing output. For unchanged autorelease components (upstream or local), +the flag records host `date -Is` output in `.no-change-rebuild`. +Upstream non-autorelease rebuilds proceed without a timestamp marker. +Unchanged local non-autorelease components must manually update their release and changelog, +even with `--no-change-rebuild`. Comparison +ignores this root marker in both existing output and the working upstream checkout. +Lockless failures never write `RENDER_FAILED`. Before replacement, output is +untouched; failures after replacement starts clear the component directory and +use Git to restore committed output, or leave no directory if none existed. +Recovery discards uncommitted changes in that directory and keeps no backups. +Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. +The package name must match. The new NEVR must be equal or higher for a final autorelease spec; +non-autorelease requires a higher NEVR. Invalid values fail with both NEVRs shown. +Validation errors leave existing output untouched and fail even without `--fail-on-error`. +First renders, `--check-only`, and dry runs skip this replacement-time check. +`--no-change-rebuild` never bypasses the NEVR check. +The comparison ignores the root `.git` without traversing or removing it. + +Changelog handling precedes release preparation and is independent of version +changes. Static specs retain the committed inline `%changelog` section. +An initial `%autorelease` render generates the complete upstream changelog on +the host. Legacy-marker output supplies its committed inline `%changelog` body +as the working `changelog`, removing leading and trailing `##` comment lines +while preserving interior comments and whitespace. Modern autorelease output +replaces the working `changelog` with the committed file byte-for-byte. +A missing committed `changelog` fails rendering. + +Before overlays, rendering compares the upstream spec's `Version:` with the +committed spec, independently of upstream pin changes. +Static release carryover runs only when committed output exists and the version +is unchanged. Otherwise the working static release +and `baserelease` definition are unchanged. +There is no later restoration of the original upstream release. + +For static specs, release carryover checks the committed `Release:` value. If it uses +the `baserelease` macro, rendering copies the committed `%global baserelease` +or `%define baserelease` line over the upstream definition without changing +the upstream `Release:` value. Missing or ambiguous definitions fail rendering. +Otherwise, it copies the committed `Release:` value. No temporary release-bump +probe is performed. An initial `%autorelease` render calculates its base release +on the host. Initial and legacy autorelease handling do not depend on version changes. +For existing output +marked by a line starting with `## RPMAUTOSPEC:` and containing `autorelease`, +an `%autorelease` upstream spec inherits the leading integer of the committed, +macro-expanded Release as `-b`. Expansion uses the committed files on the host. +Without that legacy marker, an existing `%autorelease` spec carries its +base only when the version is unchanged. For unchanged versions, it carries the +committed `-b` value into the working macro; an omitted `-b` stays omitted, +removing any working base option rather than adding `-b 1`. +An invalid numeric base fails release carryover, which requires no host release query. +When the version changes, rendering calculates the upstream numeric base and sets `-b`, +as for initial output. The committed changelog is still carried over, not regenerated. + +A calculated base of `1` omits `-b`, removing any working base option while preserving +other macro options. This includes legacy expansion. Unchanged-version modern output +still carries an explicitly committed `-b 1` as-is. + +The final host +`rpmdev-bumpspec -c` uses the exact `--changelog-comment` value or the cleaned +editor-provided entry. There is no default comment. +Upstream changes are prefilled in the editor, not appended after editing. +All prepared dist-git files except `.git` are retained, +without file filtering or `spectool` queries. Synthetic history and its +legacy release adjustment are skipped during lockless rendering. +Lockless rendering never uses mock or `rpmautospec process-distgit`; the output +retains `%autorelease`, `%autochangelog`, and other RPM macros unexpanded, and +preserves the `changelog` sidecar. + +When the last render's configuration has no `spec.upstream-commit`, its +committed lock file can supply the old pin if that commit's top-level +`lockless-components.txt` is absent or does not list the component. The current +render still requires a configured pin. + +Rendering then replaces `{{ .RenderedSpecsDir }}///` and +automatically commits that directory. The first commit also includes all staged +changes, unstaged tracked changes, and non-ignored untracked files outside the +rendered output directory. Other rendered paths are excluded even if staged; +later commits include only their component's dist-git changes. +The message is the supplied changelog comment or the entry collected through +`$EDITOR` or a local editor before output replacement. There is no default. +With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. +Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. +Autorelease skips `rpmdev-bumpspec`; missing messages are collected before output replacement. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. +Provide `--changelog-comment` for unattended rendering. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +Upstream non-autorelease still needs `--changelog-comment` even with `--no-git-commit`. +`--no-git-commit` skips automatic staging and committing, but still renders and +bumps non-autorelease specs. +It never creates empty commits. Files outside rendered output may remain staged +if committing fails; component recovery does not reset them. Configure +Git author identity and install host `rpmdevtools`, `rpm-build`, and `rpmautospec` +as needed. `--check-only` never replaces output, creates rebuild timestamps, or commits. + +For source-resolution changes, run `refresh-upstream-commit` first and review the +generated pin, then follow the commit-before-render sequence above. Do not amend +the input commit to absorb the automatically generated render commit. Local +components retain their previous rendering and manual output-commit workflow. + +Generated by `azldev docs agent`; do not hand-edit. diff --git a/internal/app/azldev/agentskill/content/remove-component.md.tmpl b/internal/app/azldev/agentskill/content/remove-component.md.tmpl index f1d1c902a..369ecb948 100644 --- a/internal/app/azldev/agentskill/content/remove-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/remove-component.md.tmpl @@ -23,8 +23,9 @@ Check for **reverse dependencies** first: if other components `BuildRequires` or `Requires` this one, removing it breaks their builds. Search the tree for the component name. Identify its binary subpackage names from the rendered output under `{{ .RenderedSpecsDir }}///`, then search for those names too. -`azldev comp query` currently supports local spec sources only, not the usual upstream -spec source. +`azldev comp query` reads rendered specs for either source type. Locked components +use working-tree rendered files; lockless components use committed rendered +dist-git history processed with rpmautospec. ## Steps diff --git a/internal/app/azldev/agentskill/content/update-component.md.tmpl b/internal/app/azldev/agentskill/content/update-component.md.tmpl index a8d158b77..d3ca6690f 100644 --- a/internal/app/azldev/agentskill/content/update-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/update-component.md.tmpl @@ -11,6 +11,10 @@ an input fingerprint computed from the component's render inputs — its TOML co overlays, the pinned upstream commit, and the distro release version. If any of those change, the lock is stale. +This workflow applies only to components not listed in `lockless-components.txt` +beside the top-level `azldev.toml`. Listed components use the +`azldev-refresh-upstream-commit` skill instead; `comp update` skips them. + ## When to run `update` | Situation | Run `update`? | diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl index c1f87f153..67dbbfccd 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/azldev.md.tmpl @@ -33,6 +33,85 @@ images assemble RPMs into bootable artifacts. ## Common commands +`azldev comp make-lockless` adds selected exact names to the top-level +`lockless-components.txt` if needed, then runs the refresh pipeline. +The list is written in lexical order, one name per line without blank lines or duplicates. +Existing names and lock files are retained. Already-listed components +are still refreshed; a refresh failure leaves the list updated. + +Before preparing any lockless component, rendering checks staged changes, +unstaged tracked changes, and untracked files outside `project.rendered-specs-dir` +and in selected components' dist-git directories. Other rendered changes are ignored. +Git-ignored files are excluded. +Use `--allow-uncommitted` for intentional dirty renders; `--check-only` and +`--no-git-commit` do not bypass this check. The override does not change +committed-history carryover or failure recovery. + +For upstream components, rendering prepares the changelog before Release, +then writes macros, applies overlays, and updates `sources`. +Version detection and carryover use the upstream spec before overlays. +Overlays can override the prepared release and changelog. +Non-autorelease rendering then runs host `rpmdev-bumpspec`. +A temporary process-local RPM wrapper sets `_sourcedir` and `_specdir` to the spec directory +without changing `HOME` or editing the user's `.rpmmacros`; it is removed after the invocation, +including on failure. +Release edits preserve the existing spaces and tabs after `Release:`. +Rendering compares output before the final release bump; `--check-only` stops before that bump. +Unchanged lockless renders require `--no-change-rebuild`; otherwise rendering errors +without replacing existing output. For unchanged autorelease components (upstream or local), +the flag records host `date -Is` output in `.no-change-rebuild`. +Upstream non-autorelease rebuilds proceed without a timestamp marker. +Unchanged local non-autorelease components must manually update their release and changelog, +even with `--no-change-rebuild`. Comparison +ignores this root marker in both existing output and the working upstream checkout. +Lockless failures never write `RENDER_FAILED`. Before replacement, output is +untouched; failures after replacement starts clear the component directory and +use Git to restore committed output, or leave no directory if none existed. +Recovery discards uncommitted changes in that directory and keeps no backups. +Before removing existing lockless output, host `rpmspec` compares source-package NEVRs. +The package name must match. The new NEVR must be equal or higher for a final autorelease spec; +non-autorelease requires a higher NEVR. Invalid values fail with both NEVRs shown. +Validation errors leave existing output untouched and fail even without `--fail-on-error`. +First renders, `--check-only`, and dry runs skip this replacement-time check. +`--no-change-rebuild` never bypasses the NEVR check. +It replaces and automatically commits the component output directory. +The first commit also includes all staged changes, unstaged tracked changes, +and non-ignored untracked files outside the rendered output directory. +Other rendered paths are excluded even if staged; later commits include only +their component's dist-git changes. +`--changelog-comment` supplies the exact `rpmdev-bumpspec -c` value and commit +message, with no default. If omitted, `$EDITOR` or a local editor collects the +entry using a template with two blank lines and `#`-commented instructions +when no upstream changes exist. Otherwise, line 1 is blank for the summary; +non-autorelease changes start on line 2, while autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. +Autorelease skips `rpmdev-bumpspec`; missing messages are collected before output replacement. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Provide `--changelog-comment` for unattended rendering. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +Upstream non-autorelease still needs `--changelog-comment` even with `--no-git-commit`. +`--no-git-commit` skips automatic staging and committing, but still renders and +bumps non-autorelease specs. +Edited upstream changes are not appended again; explicit flag messages retain the upstream Git body. +Commit refreshed TOML before rendering; configure Git author identity and install +`rpmdevtools`, `rpm-build`, and `rpmautospec` as needed. +Lockless rendering never uses mock or `rpmautospec process-distgit`; RPM macros +remain unexpanded. Lockless rendering retains all prepared dist-git files except `.git`, +without file filtering or `spectool` queries. Synthetic history +and its legacy release adjustment are skipped during rendering. + +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, while retaining other TOML sidecars. +They participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message; otherwise +rendering collects it before output replacement, with commit-only instructions for non-autorelease. +Local components with `--no-git-commit` never prompt. +`--check-only` never creates project commits. + Top-level commands in this build: {{ range .TopLevelCommands }}- `azldev {{ .Name }}` — {{ .Short }} @@ -46,12 +125,44 @@ Everyday tasks (add `-q -O json` when scripting): | Add a component | `azldev comp add` | | Build a component | `azldev comp build -p ` | | Render specs | `azldev comp render -p ` (or `-a`) | +| Extract committed dist-git history | `azldev comp dist-git-repo -o ` | | Refresh an upstream commit | `azldev comp refresh-upstream-commit -p ` | | List / build images | `azldev image list` / `azldev image build` | `comp` is an alias for `component`. Always confirm current syntax with `azldev --help`. +`comp dist-git-repo` requires exactly one component name, `--output-dir` (`-o`), +and git-filter-repo. It clones the local project's current branch with +`--single-branch --no-local`, then filters +`{{ .RenderedSpecsDir }}//` into the repository root. +Only committed rendered files/history are exported; it does not fetch upstream. +Existing output requires `--force`, which deletes it after preflight checks. + +For lockless components, `comp query` extracts committed rendered history into +temporary dist-git repositories and processes it with rpmautospec in Mock before +querying. Uncommitted rendered edits are ignored; locked queries still read the +working tree. Lockless `comp prepare-sources` writes directly to `--output-dir` +using render-style preparation. Current TOML and file inputs use the working tree, +including modified and untracked files. Release/changelog baselines use only +committed component dist-git files; edits in the existing dist-git directory are +ignored. Nonempty output requires `--force`; failures can +leave partial output. It downloads/generates sources before inline macro setup and +overlays. Lockless specs use `%define` entries, with no separate macros file, +load directive, or generated `Source9999`; locked specs retain their macros file. +Archive overlays run before spec and +loose-file overlays. It does not compare +rendered output, apply a final render release bump, or commit. It reads render history from +`project.rendered-specs-dir` using `/`, never +from `--output-dir`. It never generates synthetic history. `--without-git` +removes `.git` after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Skipping overlays also skips inline macro setup and `sources` metadata updates; +skipping sources also skips archive overlays. Missing hashes cannot be computed +when source acquisition is skipped. +Use `comp dist-git-repo` for committed history extraction. +`comp history` continues to skip lockless components. + The hidden `advanced` group (`adv`) contains specialist integrations such as MCP and mock helpers; it is intentionally omitted from normal help. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index cac327d99..436275a42 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -23,6 +23,26 @@ progress — use it only for inner-loop builds you expect to succeed. ## The inner loop +For lockless components, build uses committed rendered dist-git history extracted +with git-filter-repo; render and commit first. It fetches configured sources through +the source manager (including download/custom origins), then remaining lookaside payloads, and +builds the SRPM with Mock's rpmautospec plugin (Mock 5.3+), then builds binary RPMs. +Build macros are inline `%define` entries in the committed spec, not passed again +as Mock command-line overrides. Lockless rendering adds no macros file, load +directive, or `Source9999` entry. Locked components retain their macros file. +Render and commit macro changes before building. +Uncommitted rendered edits are ignored. `--without-git` and `--rpmdev-bumpspec` +are rejected for lockless builds; `--no-check` is forwarded normally. +Lockless `comp prepare-sources` writes directly to `--output-dir` and downloads/generates +sources before inline macro setup, then applies archive overlays before spec and +loose-file overlays. History comes from `project.rendered-specs-dir`, not +`--output-dir`; no synthetic history is generated. `--without-git` removes `.git` +after successful preparation, preserving it on failure. +It supports `--skip-overlays`, `--skip-sources`, and `--allow-no-hashes`. +Nonempty output requires `--force`; failures can leave partial output. Use `comp dist-git-repo` +for committed history extraction. +Locked builds retain their existing source preparation and flag behavior. + investigate → modify → render → build → test → inspect | Step | Command | @@ -41,6 +61,12 @@ two trees). Builds can be slow — set generous timeouts. Finalize with `azldev comp refresh-upstream-commit -p ` before opening a PR (see the `azldev-refresh-upstream-commit` skill). +For all lockless components, uncommitted tracked changes or untracked files outside +`project.rendered-specs-dir` or in selected components' dist-git directories require +`comp render --allow-uncommitted`. Other rendered changes are ignored. +Git-ignored files are excluded; the override does not change committed-history +carryover or failure recovery. Commit inputs before the final render. + ## Debugging build failures 1. **Render error mentioning a non-standard `Release` tag** — a release-calculation diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 0e33a9e6d..fcc91639e 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -31,7 +31,7 @@ Add sub-tables only for what you change. The fields you will reach for most: | `overlays` / `overlay-files` | targeted spec/source edits (see the `azldev-overlays` skill) | | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | -| `render.skip-file-filter` | rendering edge-case escape hatch | +| `render.skip-file-filter` | locked-rendering edge-case escape hatch | ## Spec source @@ -86,9 +86,44 @@ calculation = "manual" ## Render configuration -`render.skip-file-filter = true` keeps all source and patch files during render. -azldev normally prunes files not referenced by the rendered spec; set this only for +Upstream rendering runs host `rpmdev-bumpspec -c` only for non-autorelease specs +and automatically commits the component output directory. Missing +`--changelog-comment` opens `$EDITOR` or a local editor before output replacement, +including for autorelease commits. +Autorelease with `--no-git-commit` never prompts. Blank/comment-only edited messages fail rendering. +The final Git commit never prompts; its first message line is the summary, and remaining lines are body. +Required message prompts require both stdin and stdout to be TTYs; otherwise rendering fails. +Non-interactive renders must use `--changelog-comment` and/or `--no-git-commit`. +With no upstream changes, the editor starts with two blank lines and `#`-commented instructions. +Otherwise, non-autorelease changes start on line 2; autorelease puts `...` on line 2 +and starts changes on line 3. Each change line is prefixed with `- ` and is editable. +Blank lines and lines starting with `#` are removed, with no trailing newline; +the cleaned text is reused for the Git message. Explicit flag values remain unchanged. +Commit input TOML first, install host `rpmdevtools`, `rpm-build`, and `rpmautospec` as +needed, and configure Git author identity. Lockless rendering never uses mock +or `rpmautospec process-distgit`; RPM macros remain unexpanded and the +`changelog` sidecar is preserved. Synthetic history and its legacy release +adjustment are skipped during rendering. +`--check-only` never creates project commits. +Local lockless components skip upstream carryover; Release and changelog remain unchanged. +They exclude the TOML defining the local spec path, retain other TOML sidecars, +and participate in unchanged-render checks and automatic commits. +For local components, `--changelog-comment` supplies only the Git message. +Local components with `--no-git-commit` never prompt. +The first automatic commit also includes pending repository changes outside the +rendered output directory; later commits contain only their component output. + +azldev reads `lockless-components.txt` beside the top-level `azldev.toml`. +Listed component names use the lockless workflow for every command; +all others use lock files. A missing or empty list means all components use +locks. List every component to use a fully lockless workflow. +Use `comp refresh-upstream-commit` for lockless components. + +`render.skip-file-filter = true` keeps all source and patch files during locked render. +Locked rendering normally prunes files not referenced by the rendered spec; set this only for the rare spec whose `Source`/`Patch` filenames use macros the filter cannot expand. +Lockless rendering always retains all prepared dist-git files except `.git`, +without file filtering or `spectool` queries; this setting has no effect on it. ## File organization diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl deleted file mode 100644 index ecd6162d5..000000000 --- a/internal/app/azldev/agentskill/content/withoutlockfile/refresh-upstream-commit.md.tmpl +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: {{ .Name }} -description: {{ printf "%q" .Description }} ---- - -# Refresh component upstream commits - -`azldev comp refresh-upstream-commit` (`comp` is an alias for `component`) resolves one or more -components at the distro snapshot and writes normal component TOML under -`{{ .UpstreamCommitsDir }}/`. - -The project config must include `{{ .UpstreamCommitsDir }}/*.toml` **before** -the component-specific TOML configuration so subsequent commands load the -generated commit pins and the component definition supplies the remaining -`spec` fields. If `--upstream-commits-dir` selects another directory, change -the include pattern to match while preserving that ordering. - -The command uses the fully merged component configuration. It resolves and -writes a pin only when the selected component's effective `spec.type` is -`upstream`; for other source types, it removes any existing generated pin -without contacting an upstream provider. -It loads configuration permissively so a stale generated pin cannot block -cleanup after a component is removed or converted to another source type. -Other configuration validation failures are still reported as warnings. - -## When to run `refresh-upstream-commit` - -| Situation | Run `refresh-upstream-commit`? | -| --- | --- | -| Adding a new upstream component | **Yes** — first, to generate its upstream commit TOML | -| Finalizing overlay, build config, or metadata changes | No — these do not affect the resolved commit | -| Changing source resolution (commit pin, upstream distro/version, or snapshot) | **Yes** — also mid-workflow (see below) | -| Iterating on overlays / build config / metadata | No — `render` alone is enough while iterating | -| Just reading or building existing components | No | - -Refresh a single component with `-p `. Use `-a` (all components) only for -coordinated mass refreshes (e.g. a new distro snapshot) or when investigating -commit drift across many components — it is slow. For day-to-day work use `-p`. -Add `-O json` for machine-readable output when debugging. - -## Finalizing component changes - -For edits that do not change source resolution, render and commit the result. -The generated upstream commit TOML does not change: - -```sh -azldev comp render -p -git add \ - {{ .RenderedSpecsDir }}/// -git commit -m "fix(): ..." -``` - -Synthetic dist-git history is derived from committed upstream-commit TOML changes. Changes -that do not refresh the upstream commit do not add a synthetic history entry or -trigger an automatic release bump. - -## Changing source resolution - -A source-resolution change follows the same rule, using one commit followed by a -post-render amend: - -1. Change the upstream distro/version or snapshot, then `azldev comp refresh-upstream-commit -p `; sanity-check `{{ .UpstreamCommitsDir }}/.toml`. -2. `azldev comp render -p ` — the spec body now tracks the newly resolved source. `%changelog` / `Release:` still reflect the previous source; that is expected until you commit. -3. Iterate on overlays / patches / build config as the new source requires, re-rendering after each change. Re-run `refresh-upstream-commit` only if you change a source-resolution input again. -4. Stage and commit all component inputs changed above with the refreshed TOML and rendered output: - - ```sh - git add \ - {{ .UpstreamCommitsDir }}/.toml \ - {{ .RenderedSpecsDir }}/// - git commit -m "feat(): refresh upstream source" - ``` -5. `azldev comp render -p ` — `%changelog` / `Release:` now reflect the new commit TOML. -6. `git add {{ .RenderedSpecsDir }}///`, then `git commit --amend --no-edit` so the source change and rendered output land together. - -Generated by `azldev docs agent`; do not hand-edit. Generated for azldev version `{{ .Version }}`. diff --git a/internal/app/azldev/agentskill/doc.go b/internal/app/azldev/agentskill/doc.go index 49f193bcf..9d5e297a3 100644 --- a/internal/app/azldev/agentskill/doc.go +++ b/internal/app/azldev/agentskill/doc.go @@ -62,9 +62,8 @@ // // # Modes // -// A [Catalog] selects the content for one of azldev's two modes. The default mode -// documents the lock-file workflow; the mode selected by the global -// '--without-lockfile' flag documents the generated upstream-commit workflow. Skills, +// The CLI always uses the default [Catalog], which documents both per-component +// workflows. An internal all-lockless variant is also available. Skills, // instruction files, and emitted layout are shared; templates under // 'content/withoutlockfile' replace the same-named default template, and a small // registry replaces the skills whose subject differs. diff --git a/internal/app/azldev/app.go b/internal/app/azldev/app.go index f9cfda1ae..0932e5855 100644 --- a/internal/app/azldev/app.go +++ b/internal/app/azldev/app.go @@ -11,7 +11,6 @@ import ( "os" "os/signal" "path/filepath" - "strconv" "strings" "time" @@ -54,7 +53,6 @@ type App struct { commandPermissiveConfig bool configFiles []string colorMode ColorMode - withoutLockfile bool // Root command for the CLI. cmd cobra.Command @@ -191,8 +189,6 @@ func (app *App) registerGlobalFlags() { "output colorization mode {always, auto, never}") app.cmd.PersistentFlags().BoolVar(&app.permissiveConfigParsing, "permissive-config", false, "do not fail on unknown fields in TOML config files") - app.cmd.PersistentFlags().BoolVar(&app.withoutLockfile, "without-lockfile", false, - "preview: track resolved upstream commits in generated config instead of lock files") } // addAdvancedCommandHint embeds a hint about the hidden "advanced" command group @@ -426,7 +422,6 @@ func (a *App) initializeEnvOptions() *EnvOptions { envOptions.Interfaces.FileSystemFactory = a.fsFactory envOptions.Interfaces.OSEnvFactory = a.osEnvFactory envOptions.DryRunnable = NewAppDryRunnable(a.dryRun) - envOptions.WithoutLockfile = a.withoutLockfile return &envOptions } @@ -488,68 +483,16 @@ func setEventListener(stdioLogger *slog.Logger, quiet, verbose bool, envOptions return nil } -// PreParseGlobalFlags hand-parses the global flags that must be known before commands -// are registered and configuration is loaded. It is safe to call more than once with the +// PreParseGlobalFlags hand-parses the global flags needed to load configuration. +// It is safe to call more than once with the // same arguments; each call fully recomputes the pre-parsed state. -// -// Command registration is mode-sensitive, so the CLI entry point calls this before it -// registers commands; [App.Execute] calls it again so that an App executed directly -// (e.g. from a test) behaves identically. func (a *App) PreParseGlobalFlags(args []string) { // Reset accumulating state so repeated calls are idempotent. a.configFiles = nil - a.withoutLockfile = parseWithoutLockfileFlag(args) a.handParseConfigFlags(args) } -// WithoutLockfile reports whether the preview lock-file-free mode was requested via the -// global '--without-lockfile' flag. Valid only after [App.PreParseGlobalFlags] has run. -func (a *App) WithoutLockfile() bool { - return a.withoutLockfile -} - -// withoutLockfileFlagName is the global flag that selects lock-file-free mode. -const withoutLockfileFlagName = "--without-lockfile" - -// parseWithoutLockfileFlag hand-parses the global '--without-lockfile' boolean flag. -// -// Only exact '--without-lockfile' and '--without-lockfile=' tokens are -// recognized, and scanning stops at the '--' terminator so that positional text -// (for example a mock command line) is never mistaken for the flag. An invalid or -// missing value is treated the same way cobra treats it: '--without-lockfile' alone -// enables the mode, and an unparseable '=' leaves the mode disabled so the -// final cobra parse reports the error. -func parseWithoutLockfileFlag(args []string) bool { - withoutLockfile := false - - for _, arg := range args { - if arg == "--" { - break - } - - if arg == withoutLockfileFlagName { - withoutLockfile = true - - continue - } - - value, found := strings.CutPrefix(arg, withoutLockfileFlagName+"=") - if !found { - continue - } - - parsed, err := strconv.ParseBool(value) - if err != nil { - return false - } - - withoutLockfile = parsed - } - - return withoutLockfile -} - // Hand-parses a few critical configuration flags from the command line -- just enough to // find the project and load configuration so we can properly use cobra facilities to // parse the full command line. @@ -629,7 +572,7 @@ func (a *App) findAndLoadConfig(tempDirPath string, extraConfigFiles []string) ( tempDirPath, extraConfigFiles, a.permissiveConfigEnabled(), - a.withoutLockfile, + false, /*withoutLockfile*/ ) if err != nil { return projectDir, config, fmt.Errorf("failed to load project configuration:\n%w", err) diff --git a/internal/app/azldev/app_test.go b/internal/app/azldev/app_test.go index 64ac1397b..1b5af8648 100644 --- a/internal/app/azldev/app_test.go +++ b/internal/app/azldev/app_test.go @@ -238,66 +238,31 @@ func TestApp_ConcurrencyOption(t *testing.T) { } } -func TestApp_WithoutLockfileOption(t *testing.T) { +func TestApp_RejectsRemovedWithoutLockfileFlag(t *testing.T) { testCases := []struct { - name string - args []string - expected bool + name string + args []string }{ - {name: "absent", args: []string{"test-cmd"}, expected: false}, - {name: "bare", args: []string{"--without-lockfile", "test-cmd"}, expected: true}, - {name: "explicit true", args: []string{"--without-lockfile=true", "test-cmd"}, expected: true}, - {name: "explicit false", args: []string{"--without-lockfile=false", "test-cmd"}, expected: false}, - {name: "after command", args: []string{"test-cmd", "--without-lockfile"}, expected: true}, - {name: "positional lookalike", args: []string{"test-cmd", "--", "--without-lockfile"}, expected: false}, - {name: "prefix lookalike", args: []string{"test-cmd", "--without-lockfiles"}, expected: false}, + {name: "bare", args: []string{"--without-lockfile", "test-cmd"}}, + {name: "explicit true", args: []string{"--without-lockfile=true", "test-cmd"}}, + {name: "explicit false", args: []string{"--without-lockfile=false", "test-cmd"}}, + {name: "after command", args: []string{"test-cmd", "--without-lockfile"}}, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { app := createTestApp(t) - - app.PreParseGlobalFlags(testCase.args) - - assert.Equal(t, testCase.expected, app.WithoutLockfile()) - }) - } -} - -func TestApp_WithoutLockfileOption_ReachesEnv(t *testing.T) { - testCases := []struct { - name string - args []string - expected bool - }{ - {name: "default", args: []string{"test-cmd"}, expected: false}, - {name: "enabled", args: []string{"--without-lockfile", "test-cmd"}, expected: true}, - {name: "disabled", args: []string{"--without-lockfile=false", "test-cmd"}, expected: false}, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - app := createTestApp(t) - ran := false cmd := &cobra.Command{ Use: "test-cmd", - RunE: func(cmd *cobra.Command, _ []string) error { - env, err := azldev.GetEnvFromCommand(cmd) - require.NoError(t, err) - - assert.Equal(t, testCase.expected, env.WithoutLockfile()) - + Run: func(_ *cobra.Command, _ []string) { ran = true - - return nil }, } - app.AddTopLevelCommand(cmd) - assert.Zero(t, app.Execute(testCase.args)) - assert.True(t, ran) + assert.NotZero(t, app.Execute(testCase.args)) + assert.False(t, ran) }) } } @@ -306,7 +271,7 @@ func TestApp_WithoutLockfileOption_ReachesEnv(t *testing.T) { // registration) and by Execute; repeated calls must not accumulate state. func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { app := createTestApp(t) - args := []string{"--without-lockfile", "--config-file", "extra.toml", "test-cmd"} + args := []string{"--config-file", "extra.toml", "test-cmd"} ran := false cmd := &cobra.Command{ @@ -315,7 +280,7 @@ func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { env, err := azldev.GetEnvFromCommand(cmd) require.NoError(t, err) - assert.True(t, env.WithoutLockfile()) + assert.False(t, env.WithoutLockfile()) ran = true @@ -327,8 +292,6 @@ func TestApp_PreParseGlobalFlags_Idempotent(t *testing.T) { app.PreParseGlobalFlags(args) app.PreParseGlobalFlags(args) - assert.True(t, app.WithoutLockfile()) - // Execute pre-parses the same arguments a third time; a command that only // accumulated config files would fail to load its (nonexistent) extra file. assert.Zero(t, app.Execute(args)) diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 2c9f99171..a2229d197 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -80,11 +80,11 @@ type ComponentBuildResults struct { RPMs []RPMResult `json:"rpms" table:"-"` } -func buildOnAppInit(app *azldev.App, parent *cobra.Command) { - parent.AddCommand(NewBuildCmd(cmdOptionsForApp(app)...)) +func buildOnAppInit(_ *azldev.App, parent *cobra.Command) { + parent.AddCommand(NewBuildCmd()) } -func NewBuildCmd(opts ...CmdOption) *cobra.Command { +func NewBuildCmd() *cobra.Command { // Fill out options defaults. options := &ComponentBuildOptions{ BuildEnvPolicy: BuildEnvPreserveOnFailure, @@ -105,6 +105,12 @@ are placed in structured subdirectories under the project output directory: out/rpms/ - binary RPMs with no channel configured (or channel="none") out/rpms// - binary RPMs moved to their configured publish channel subdirectory +For lockless components, build instead extracts committed rendered dist-git +history using git-filter-repo, downloads lookaside sources, and builds the SRPM +with mock's rpmautospec plugin (mock 5.3+). Uncommitted rendered changes are +ignored. '--without-git' and '--rpmdev-bumpspec' are unsupported for lockless +components. '--no-check' skips the binary build's package checks normally. + The publish channel for each package is resolved from the project's package configuration (package groups and per-component overrides). See 'azldev package' for details. @@ -132,7 +138,7 @@ builds can consume.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().BoolVarP(&options.ContinueOnError, "continue-on-error", "k", false, "Continue building when some components fail") cmd.Flags().BoolVar(&options.NoCheck, "no-check", false, "Skip package %check tests") @@ -242,6 +248,10 @@ func buildComponent( mockProcessor *sources.MockProcessor, options *ComponentBuildOptions, ) (results ComponentBuildResults, err error) { + if env.ComponentWithoutLockfile(component.GetName()) { + return buildLocklessComponent(env, component, workDirFactory, options) + } + // Resolve the effective distro for this component before creating the source manager. distro, err := sourceproviders.ResolveDistro(env, component) if err != nil { @@ -275,7 +285,7 @@ func buildComponent( var preparerOpts []sources.PreparerOption if !options.WithoutGitRepo { - preparerOpts = append(preparerOpts, gitRepoPreparerOptions(env, distro)...) + preparerOpts = append(preparerOpts, gitRepoPreparerOptions(env, distro, component.GetName())...) } if options.RPMDevBumpspec { diff --git a/internal/app/azldev/cmds/component/build_lockless.go b/internal/app/azldev/cmds/component/build_lockless.go new file mode 100644 index 000000000..1e7ab043a --- /dev/null +++ b/internal/app/azldev/cmds/component/build_lockless.go @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "fmt" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/downloadsources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/componentbuilder" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/defers" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +func buildLocklessComponent( + env *azldev.Env, component components.Component, factory *workdir.Factory, options *ComponentBuildOptions, +) (results ComponentBuildResults, err error) { + name := component.GetName() + if options.WithoutGitRepo || options.RPMDevBumpspec { + return results, fmt.Errorf("lockless component %#q does not support '--without-git' or '--rpmdev-bumpspec'", name) + } + + dir, err := factory.Create(name, "dist-git") + if err != nil { + return results, fmt.Errorf("creating dist-git work directory:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), dir, &err) + + if _, err := CreateComponentDistGitRepo(env, &DistGitRepoOptions{ + Component: name, OutputDir: dir, Force: true, + }); err != nil { + return results, err + } + + if err := downloadDistGitSources(env, component, dir); err != nil { + return results, err + } + + buildEnv, err := workdir.MkComponentBuildEnvironment(env, factory, component.GetConfig(), "build", + options.MockConfigOpts) + if err != nil { + return results, fmt.Errorf("creating lockless build environment:\n%w", err) + } + defer defers.HandleDeferError(func() error { + if !options.BuildEnvPolicy.ShouldPreserve(err == nil) { + return buildEnv.Destroy(env) + } + + return nil + }, &err) + + builder := componentbuilder.New(env, env.FS(), env, nil, buildEnv, factory) + builder.UseDistGitSources(dir) + + repos := append([]string{}, options.LocalRepoPaths...) + if options.LocalRepoWithPublishPath != "" { + repos = append(repos, options.LocalRepoWithPublishPath) + } + + return buildComponentUsingBuilder(env, component, builder, repos, options.SourcePackageOnly, + options.NoCheck, options.LocalRepoWithPublishPath) +} + +func downloadDistGitSources(env *azldev.Env, component components.Component, dir string) error { + if len(component.GetConfig().SourceFiles) > 0 { + distro, err := sourceproviders.ResolveDistro(env, component) + if err != nil { + return fmt.Errorf("resolving source distro:\n%w", err) + } + + manager, err := sourceproviders.NewSourceManager(env, distro) + if err != nil { + return fmt.Errorf("creating source manager:\n%w", err) + } + + if err := manager.FetchFiles(env, component, dir); err != nil { + return fmt.Errorf("fetching configured dist-git sources:\n%w", err) + } + } + + path := filepath.Join(dir, "sources") + + exists, err := fileutils.Exists(env.FS(), path) + if err != nil { + return fmt.Errorf("checking dist-git 'sources' manifest:\n%w", err) + } + + if !exists { + return nil + } + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return fmt.Errorf("reading dist-git 'sources' manifest:\n%w", err) + } + + lines, err := fedorasource.ReadSourcesFile(string(content)) + if err != nil { + return fmt.Errorf("parsing dist-git 'sources' manifest:\n%w", err) + } + + for _, line := range lines { + if line.Entry == nil { + continue + } + + exists, err := fileutils.Exists(env.FS(), filepath.Join(dir, line.Entry.Filename)) + if err != nil { + return fmt.Errorf("checking dist-git source %#q:\n%w", line.Entry.Filename, err) + } + + if !exists { + if err := downloadsources.DownloadSources(env, &downloadsources.DownloadSourcesOptions{ + Directory: dir, ComponentName: component.GetName(), + }); err != nil { + return fmt.Errorf("downloading dist-git sources:\n%w", err) + } + + break + } + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/build_lockless_internal_test.go b/internal/app/azldev/cmds/component/build_lockless_internal_test.go new file mode 100644 index 000000000..998da663e --- /dev/null +++ b/internal/app/azldev/cmds/component/build_lockless_internal_test.go @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/h2non/gock" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/workdir" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/require" +) + +func TestDownloadDistGitSources_ConfiguredOrigins(t *testing.T) { + const ( + filename = "extra-source.tar.gz" + payload = "hello world" + digest = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ) + + for _, testCase := range []struct { + name string + manifest bool + disableOrigins bool + custom bool + wantError string + }{ + {name: "download with manifest", manifest: true}, + {name: "download without manifest"}, + {name: "origins disabled", manifest: true, disableOrigins: true, wantError: "disable-origins is enabled"}, + {name: "custom origin dispatched", manifest: true, custom: true, wantError: "generation script"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + distro := env.Config.Distros["test-distro"] + distro.LookasideBaseURI = "" + distro.DisableOrigins = testCase.disableOrigins + env.Config.Distros["test-distro"] = distro + + ref := projectconfig.SourceFileReference{ + Filename: filename, Hash: digest, HashType: fileutils.HashTypeSHA256, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://origin.invalid/source"}, + } + if testCase.custom { + ref.Origin = projectconfig.Origin{Type: projectconfig.OriginTypeCustom, Script: "/project/generate.sh"} + } + + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", SourceFiles: []projectconfig.SourceFileReference{ref}, + } + dir := "/project/dist-git" + require.NoError(t, fileutils.MkdirAll(env.FS(), dir)) + + if testCase.manifest { + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(dir, "sources"), + []byte("SHA256 ("+filename+") = "+digest+"\n"), fileperms.PublicFile)) + } + + if testCase.wantError == "" { + gock.New("https://origin.invalid").Get("/source").Reply(200).BodyString(payload) + t.Cleanup(gock.OffAll) + } + + err := downloadDistGitSources(env.Env, resolveComponents(t, env, "curl")[0], dir) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + require.True(t, gock.IsDone()) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, filename)) + require.NoError(t, err) + require.Equal(t, payload, string(content)) + }) + } +} + +func TestLocklessBuildRejectsSyntheticHistoryFlags(t *testing.T) { + for _, options := range []ComponentBuildOptions{{WithoutGitRepo: true}, {RPMDevBumpspec: true}} { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + comp := resolveComponents(t, env, "curl")[0] + _, err := BuildComponent(env.Env, comp, nil, &options) + require.ErrorContains(t, err, "does not support '--without-git' or '--rpmdev-bumpspec'") + } +} + +func TestLocklessBuildUsesDistGitAndCleansUp(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "build failure"}[fail], func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = "/project/specs" + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Build: projectconfig.ComponentBuildConfig{ + With: []string{"feature"}, Without: []string{"other"}, + Defines: map[string]string{"dist": ".azl3", "custom": "working-tree-value"}, + Undefines: []string{"dist"}, + }, + } + env.CmdFactory.RegisterCommandInSearchPath("git") + env.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + var cloneDir string + + sentinel := errors.New("injected SRPM failure") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] != "git" { + return "/mock/root", nil + } + + switch cmd.Args[3] { + case "rev-parse": + if cmd.Args[4] == "--git-common-dir" { + return filepath.Join("/project", ".git"), nil + } + + return "/project", nil + case "clone": + cloneDir = cmd.Args[len(cmd.Args)-1] + require.Equal(t, []string{"--single-branch", "--no-local", "--"}, cmd.Args[4:7]) + require.NoError(t, fileutils.MkdirAll(env.FS(), cloneDir)) + case "filter-repo": + if slices.Contains(cmd.Args, "--subdirectory-filter") { + require.Equal(t, "specs/c/curl", cmd.Args[len(cmd.Args)-1]) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.spec"), + []byte("%{load:%{_sourcedir}/curl.azl.macros}\n"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, "curl.azl.macros"), + []byte("%custom committed-value\n"), fileperms.PublicFile)) + } + } + + return "", nil + } + built := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.NotContains(t, strings.Join(cmd.Args, " "), "calculate-release") + + if !slices.Contains(cmd.Args, "--buildsrpm") { + return nil + } + + built = true + + require.NotContains(t, cmd.Args, "--define") + require.NotContains(t, cmd.Args, "--with") + require.NotContains(t, cmd.Args, "--without") + require.Contains(t, strings.Join(cmd.Args, " "), "--enable-plugin rpmautospec") + require.Equal(t, cloneDir, cmd.Args[slices.Index(cmd.Args, "--sources")+1]) + macros, err := fileutils.ReadFile(env.FS(), filepath.Join(cloneDir, "curl.azl.macros")) + require.NoError(t, err) + require.Equal(t, "%custom committed-value\n", string(macros)) + + if fail { + return sentinel + } + + out := cmd.Args[slices.Index(cmd.Args, "--resultdir")+1] + + return fileutils.WriteFile(env.FS(), filepath.Join(out, "curl.src.rpm"), []byte("srpm"), fileperms.PublicFile) + } + factory, err := workdir.NewFactory(env.FS(), env.Env.WorkDir(), env.Env.ConstructionTime()) + require.NoError(t, err) + comp := resolveComponents(t, env, "curl")[0] + + results, err := BuildComponent(env.Env, comp, factory, &ComponentBuildOptions{SourcePackageOnly: true}) + if fail { + require.ErrorIs(t, err, sentinel) + } else { + require.NoError(t, err) + require.Len(t, results.SRPMPaths, 1) + } + + require.True(t, built) + + exists, err := fileutils.Exists(env.FS(), cloneDir) + require.NoError(t, err) + require.False(t, exists) + }) + } +} diff --git a/internal/app/azldev/cmds/component/changed.go b/internal/app/azldev/cmds/component/changed.go index 31226191b..3a90cb975 100644 --- a/internal/app/azldev/cmds/component/changed.go +++ b/internal/app/azldev/cmds/component/changed.go @@ -32,41 +32,22 @@ type ChangedComponentOptions struct { IncludeUnchanged bool } -func changedOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewChangedCmd(cmdOptionsForApp(app)...)) +func changedOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewChangedCmd()) } -// changedCommandLong returns the long help for 'component changed', which -// describes the comparison performed by the active mode. -func changedCommandLong(options cmdOptions) string { - if options.withoutLockfile { - return `Load the project configuration independently at two git refs and compare the -resolved component build inputs. Normal project TOML parsing is used at each -ref, including recursive includes, config merging, component defaults, and -generated upstream-commit TOMLs included by the project. - -For each selected component, the command reports whether the build inputs -changed: normalized component configuration, upstream commit or local -spec-directory contents, overlay source filenames and contents, and effective -distro release version. - -Documentation, publishing, test-selection, scheduling-hint, snapshot-time, and -checkout-path-only fields do not trigger a component change. For components -whose build inputs changed, the sourcesChange field separately compares the raw -committed sources manifests at the rendered spec directories configured by each -ref. All-component scans use the union of components in both refs, so added and -deleted components are reported without consulting the current checkout. - -This is useful for CI/CD pipelines to determine which components need to be -rebuilt or have their lookaside tarballs re-uploaded after a PR merge.` - } - +// changedCommandLong describes the per-component comparisons. +func changedCommandLong() string { return `Compare component lock files and rendered sources between two git refs to determine which components changed. For each component, reports whether its input fingerprint changed (any change). For components whose fingerprint changed, it also reports whether the rendered sources file changed (sources change). +For components listed in lockless-components.txt beside the current checkout's +top-level azldev.toml, compare resolved project build inputs at both refs instead +of stored locks. + This is useful for CI/CD pipelines to determine which components need to be rebuilt or have their lookaside tarballs re-uploaded after a PR merge. @@ -78,14 +59,13 @@ detected via lock file presence in the compared refs when using -a.` } // NewChangedCmd constructs a [cobra.Command] for the "component changed" CLI subcommand. -func NewChangedCmd(opts ...CmdOption) *cobra.Command { +func NewChangedCmd() *cobra.Command { options := &ChangedComponentOptions{} - cmdOptions := newCmdOptions(opts...) cmd := &cobra.Command{ Use: "changed", Short: "Detect which components changed between two git refs", - Long: changedCommandLong(cmdOptions), + Long: changedCommandLong(), Example: ` # Show changed components between a branch and HEAD azldev component changed --from main -a @@ -105,7 +85,7 @@ func NewChangedCmd(opts ...CmdOption) *cobra.Command { ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVar(&options.From, "from", "", "Git ref to compare from (required)") cmd.Flags().StringVar(&options.To, "to", "HEAD", "Git ref to compare to") @@ -116,9 +96,7 @@ func NewChangedCmd(opts ...CmdOption) *cobra.Command { // Hide inherited flag -- this command always skips lock validation since // it inspects historical locks at arbitrary refs. - if !cmdOptions.withoutLockfile { - _ = cmd.Flags().MarkHidden("skip-lock-validation") - } + _ = cmd.Flags().MarkHidden("skip-lock-validation") azldev.ExportAsReadOnlyMCPTool(cmd) @@ -140,10 +118,8 @@ const ( changeTypeDeleted = "deleted" ) -// ChangedComponents determines which components changed between two git refs, -// using the comparison that matches the active mode: stored lock file -// fingerprints by default, or configuration resolved at each ref when the global -// '--without-lockfile' flag is set. +// ChangedComponents compares stored fingerprints for locked components and +// resolved project inputs for lockless components. func ChangedComponents( env *azldev.Env, options *ChangedComponentOptions, ) ([]ChangedResult, error) { @@ -151,7 +127,63 @@ func ChangedComponents( return changedComponentsFromProjectConfigs(env, options) } - return changedComponentsFromLocks(env, options) + if !hasLocklessChangeSelection(env, &options.ComponentFilter) { + return changedComponentsFromLocks(env, options) + } + + locklessResults, err := changedComponentsFromProjectConfigs(env, options) + if err != nil { + return nil, err + } + + lockedOptions := *options + + lockedOptions.ComponentFilter.ComponentNamePatterns = nil + for _, pattern := range options.ComponentFilter.ComponentNamePatterns { + if !env.ComponentWithoutLockfile(pattern) { + lockedOptions.ComponentFilter.ComponentNamePatterns = append( + lockedOptions.ComponentFilter.ComponentNamePatterns, pattern, + ) + } + } + + if lockedOptions.ComponentFilter.HasNoCriteria() { + return locklessResults, nil + } + + lockedResults, err := changedComponentsFromLocks(env, &lockedOptions) + if err != nil { + return nil, err + } + + lockedResults = append(lockedResults, locklessResults...) + sort.Slice(lockedResults, func(i, j int) bool { return lockedResults[i].Component < lockedResults[j].Component }) + + return lockedResults, nil +} + +func hasLocklessChangeSelection(env *azldev.Env, filter *components.ComponentFilter) bool { + if len(env.Config().LocklessComponents) == 0 { + return false + } + + if filter.IncludeAllComponents || len(filter.ComponentGroupNames) > 0 || len(filter.SpecPaths) > 0 { + return true + } + + for name, lockless := range env.Config().LocklessComponents { + if !lockless { + continue + } + + for _, pattern := range filter.ComponentNamePatterns { + if matched, err := filepath.Match(pattern, name); matched || err != nil { + return true + } + } + } + + return false } // changedComponentsFromLocks compares component lock files and rendered sources @@ -163,13 +195,15 @@ func changedComponentsFromLocks( // the current working-tree locks may legitimately be stale. options.ComponentFilter.SkipLockValidation = true - resolver := components.NewResolver(env) - - comps, err := resolver.FindComponents(&options.ComponentFilter) + locked, err := findLockedComponentsForChanges(env, &options.ComponentFilter) if err != nil { return nil, fmt.Errorf("resolving components:\n%w", err) } + if locked.Len() == 0 && !options.ComponentFilter.IncludeAllComponents { + return []ChangedResult{}, nil + } + ctx, err := newChangedContext(env) if err != nil { return nil, err @@ -186,12 +220,12 @@ func changedComponentsFromLocks( } // Batch-read all locks at both refs. - fromLocks, err := lockfile.ReadAllAtCommit(ctx.repo, fromHash, ctx.lockRelDir) + fromLocks, err := readChangedLocks(env, ctx, fromHash) if err != nil { return nil, fmt.Errorf("reading locks at --from:\n%w", err) } - toLocks, err := lockfile.ReadAllAtCommit(ctx.repo, toHash, ctx.lockRelDir) + toLocks, err := readChangedLocks(env, ctx, toHash) if err != nil { return nil, fmt.Errorf("reading locks at --to:\n%w", err) } @@ -208,7 +242,7 @@ func changedComponentsFromLocks( } results, err := buildResults( - comps, fromLocks, toLocks, ctx, fromTree, toTree, + locked, fromLocks, toLocks, ctx, fromTree, toTree, options.IncludeUnchanged, options.ComponentFilter.IncludeAllComponents, ) if err != nil { @@ -218,6 +252,43 @@ func changedComponentsFromLocks( return results, nil } +func findLockedComponentsForChanges( + env *azldev.Env, filter *components.ComponentFilter, +) (*components.ComponentSet, error) { + resolver := components.NewResolver(env) + + comps, err := resolver.FindComponents(filter) + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + locked := components.NewComponentSet() + for _, comp := range componentsForLockMode(env, comps.Components(), false) { + locked.Add(comp) + } + + return locked, nil +} + +func readChangedLocks( + env *azldev.Env, ctx *changedContext, hash string, +) (map[string]lockfile.ComponentLock, error) { + var ignored []string + + for name, lockless := range env.Config().LocklessComponents { + if lockless { + ignored = append(ignored, name) + } + } + + locks, err := lockfile.ReadAllAtCommit(ctx.repo, hash, ctx.lockRelDir, ignored...) + if err != nil { + return nil, fmt.Errorf("reading historical component locks:\n%w", err) + } + + return locks, nil +} + // changedContext holds resolved repo state for change detection. type changedContext struct { repo *gogit.Repository diff --git a/internal/app/azldev/cmds/component/changed_test.go b/internal/app/azldev/cmds/component/changed_test.go index 0e0952ea6..bf33c9612 100644 --- a/internal/app/azldev/cmds/component/changed_test.go +++ b/internal/app/azldev/cmds/component/changed_test.go @@ -60,11 +60,9 @@ func TestChangedCmd_NoComponents(t *testing.T) { assert.Contains(t, err.Error(), "component not found") } -func TestNewChangedCmd_LongDescriptionByMode(t *testing.T) { - defaultCmd := componentcmds.NewChangedCmd() - assert.Contains(t, defaultCmd.Long, "Compare component lock files") - - withoutLockfileCmd := componentcmds.NewChangedCmd(componentcmds.WithoutLockfileFlags()) - assert.Contains(t, withoutLockfileCmd.Long, "Load the project configuration independently") - assert.NotContains(t, withoutLockfileCmd.Long, "lock file") +func TestNewChangedCmd_DocumentsPerComponentSelection(t *testing.T) { + cmd := componentcmds.NewChangedCmd() + assert.Contains(t, cmd.Long, "Compare component lock files") + assert.Contains(t, cmd.Long, "lockless-components.txt") + assert.Contains(t, cmd.Long, "compare resolved project build inputs") } diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index ce8e19ebc..0cf141267 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -6,6 +6,7 @@ package component import ( "bytes" "encoding/json" + "errors" "fmt" "log/slog" "path/filepath" @@ -27,9 +28,9 @@ import ( // This file implements 'component changed' for lock-file-free mode. Without lock // files there are no stored fingerprints to compare, so the project configuration -// is loaded independently at both refs and the resolved component build inputs are -// compared directly. The git plumbing helpers are shared with the default -// implementation in changed.go. +// is loaded independently at both refs and the resolved component build inputs and +// rendered trees are compared directly. The git plumbing helpers are shared with +// the default implementation in changed.go. // changedComponentsFromProjectConfigs loads and compares resolved project // components and rendered sources between two git refs. It is the lock-file-free @@ -84,6 +85,10 @@ func changedComponentsFromProjectConfigs( return nil, fmt.Errorf("selecting components:\n%w", err) } + names = slices.DeleteFunc(names, func(name string) bool { + return !env.ComponentWithoutLockfile(name) + }) + return buildHistoricalResults( names, fromProject, toProject, fromTree, toTree, options.IncludeUnchanged, options.ComponentFilter.IncludeAllComponents, @@ -126,6 +131,46 @@ func loadHistoricalProject( tree *object.Tree, projectRelDir string, ) (*historicalProject, error) { + historicalEnv, err := loadHistoricalEnv(env, tree, projectRelDir) + if err != nil { + return nil, err + } + + config := historicalEnv.Config() + resolver := components.NewResolver(historicalEnv) + + resolvedComponents, err := resolver.FindAllComponents() + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + result := &historicalProject{ + components: make(map[string]projectconfig.ComponentConfig, resolvedComponents.Len()), + comparisonInputs: make(map[string]componentComparisonInputs, resolvedComponents.Len()), + componentGroups: make(map[string][]string, len(config.ComponentGroups)), + } + + if err := populateHistoricalComponents( + result, historicalEnv.FS(), historicalEnv, resolvedComponents, env, + ); err != nil { + return nil, err + } + + if err := populateHistoricalGroups(result, resolver, config); err != nil { + return nil, err + } + + result.renderedSpecsRelDir, err = repoRelPath(snapshotRepoRoot, config.Project.RenderedSpecsDir) + if err != nil { + return nil, fmt.Errorf("resolving rendered specs directory:\n%w", err) + } + + return result, nil +} + +func loadHistoricalEnv( + env *azldev.Env, tree *object.Tree, projectRelDir string, +) (*azldev.Env, error) { snapshotFS := afero.NewMemMapFs() if err := copyTreeToFS(tree, snapshotFS, snapshotRepoRoot); err != nil { @@ -138,6 +183,10 @@ func loadHistoricalProject( projectDir := filepath.Join(snapshotRepoRoot, projectRelDir) + // Historical snapshots may use config fields that this azldev no longer + // supports, so always parse them permissively. + const permissive = true + loadedProjectDir, config, err := projectconfig.LoadProjectConfig( snapshotFS, env.OSEnv(), @@ -145,7 +194,7 @@ func loadHistoricalProject( false, snapshotTempDir, nil, - env.PermissiveConfigParsing(), + permissive, true, /*withoutLockfile*/ ) if err != nil { @@ -160,37 +209,9 @@ func loadHistoricalProject( FileSystemFactory: &fixedFileSystemFactory{fs: snapshotFS}, }, }) - historicalEnv.SetPermissiveConfigParsing(env.PermissiveConfigParsing()) - - resolver := components.NewResolver(historicalEnv) - - resolvedComponents, err := resolver.FindAllComponents() - if err != nil { - return nil, fmt.Errorf("resolving components:\n%w", err) - } + historicalEnv.SetPermissiveConfigParsing(permissive) - result := &historicalProject{ - components: make(map[string]projectconfig.ComponentConfig, resolvedComponents.Len()), - comparisonInputs: make(map[string]componentComparisonInputs, resolvedComponents.Len()), - componentGroups: make(map[string][]string, len(config.ComponentGroups)), - } - - if err := populateHistoricalComponents( - result, snapshotFS, historicalEnv, resolvedComponents, - ); err != nil { - return nil, err - } - - if err := populateHistoricalGroups(result, resolver, config); err != nil { - return nil, err - } - - result.renderedSpecsRelDir, err = repoRelPath(snapshotRepoRoot, config.Project.RenderedSpecsDir) - if err != nil { - return nil, fmt.Errorf("resolving rendered specs directory:\n%w", err) - } - - return result, nil + return historicalEnv, nil } func populateHistoricalGroups( @@ -224,9 +245,13 @@ func populateHistoricalComponents( fs opctx.FS, env *azldev.Env, resolvedComponents *components.ComponentSet, + selectionEnv *azldev.Env, ) error { for _, component := range resolvedComponents.Components() { project.components[component.GetName()] = *component.GetConfig() + if !selectionEnv.ComponentWithoutLockfile(component.GetName()) { + continue + } inputs, err := buildComponentComparisonInputs(fs, env, component.GetConfig()) if err != nil { @@ -544,6 +569,22 @@ func buildHistoricalResults( return nil, fmt.Errorf("comparing component %#q:\n%w", name, err) } + if result.ChangeType == changeTypeUnchanged { + fromHash, err := historicalRenderedTreeHash(fromTree, fromProject.renderedSpecsRelDir, name) + if err != nil { + return nil, fmt.Errorf("reading rendered tree for %#q at --from:\n%w", name, err) + } + + toHash, err := historicalRenderedTreeHash(toTree, toProject.renderedSpecsRelDir, name) + if err != nil { + return nil, fmt.Errorf("reading rendered tree for %#q at --to:\n%w", name, err) + } + + if fromHash != toHash { + result.ChangeType = changeTypeChanged + } + } + if result.ChangeType == changeTypeUnchanged { if includeAllComponents && !includeUnchanged { continue @@ -567,6 +608,24 @@ func buildHistoricalResults( return results, nil } +func historicalRenderedTreeHash(tree *object.Tree, renderedDir, name string) (string, error) { + dir, err := components.RenderedSpecDir(renderedDir, name) + if err != nil { + return "", fmt.Errorf("resolving rendered component directory:\n%w", err) + } + + rendered, err := tree.Tree(filepath.ToSlash(dir)) + if errors.Is(err, object.ErrDirectoryNotFound) { + return "", nil + } + + if err != nil { + return "", fmt.Errorf("reading rendered component directory:\n%w", err) + } + + return rendered.Hash.String(), nil +} + // classifyHistoricalComponent compares the serialized build-relevant inputs for a // component at two refs. func classifyHistoricalComponent( diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go index 357f3321b..e2cbc4bda 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -124,13 +124,76 @@ func TestClassifyHistoricalComponent_ExcludedFieldsUnchanged(t *testing.T) { assert.Equal(t, changeTypeUnchanged, result.ChangeType) } -func TestBuildHistoricalResults_UnchangedSkipsSourcesComparison(t *testing.T) { +func TestBuildHistoricalResults_RenderOnlyChanges(t *testing.T) { + for _, testCase := range []struct { + name string + path string + missingTree bool + reverse bool + wantChanged bool + wantSources bool + }{ + {name: "spec", path: "SPECS/c/curl/curl.spec", wantChanged: true}, + {name: "sources", path: "SPECS/c/curl/sources", wantChanged: true, wantSources: true}, + {name: "sidecar", path: "SPECS/c/curl/patches/fix.patch", wantChanged: true}, + {name: "no-change rebuild", path: "SPECS/c/curl/.no-change-rebuild", wantChanged: true}, + {name: "other component", path: "SPECS/z/zlib/zlib.spec"}, + {name: "new render", path: "SPECS/c/curl/curl.spec", missingTree: true, wantChanged: true}, + {name: "removed render", path: "SPECS/c/curl/curl.spec", missingTree: true, reverse: true, wantChanged: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + fromPath := testCase.path + if testCase.missingTree { + fromPath = "README.md" + } + + repo, fromRef, toRef := testRepoWithTwoCommits(t, + map[string][]byte{fromPath: []byte("before")}, + map[string][]byte{testCase.path: []byte("after")}, + ) + + fromTree, err := resolveTree(repo, fromRef) + require.NoError(t, err) + toTree, err := resolveTree(repo, toRef) + require.NoError(t, err) + + if testCase.reverse { + fromTree, toTree = toTree, fromTree + } + + project := &historicalProject{ + comparisonInputs: map[string]componentComparisonInputs{ + "curl": {Config: projectconfig.ComponentConfig{Name: "curl"}}, + }, + renderedSpecsRelDir: "SPECS", + } + + results, err := buildHistoricalResults( + []string{"curl"}, project, project, fromTree, toTree, false, true, + ) + require.NoError(t, err) + + if !testCase.wantChanged { + assert.Empty(t, results) + + return + } + + require.Len(t, results, 1) + assert.Equal(t, changeTypeChanged, results[0].ChangeType) + assert.Equal(t, testCase.wantSources, results[0].SourcesChange) + }) + } +} + +func TestBuildHistoricalResults_UnchangedRenderedTree(t *testing.T) { repo, fromRef, toRef := testRepoWithTwoCommits(t, map[string][]byte{ "SPECS/c/curl/sources": []byte("SHA512 (curl-1.tar.gz) = aaa"), }, map[string][]byte{ - "SPECS/c/curl/sources": []byte("SHA512 (curl-2.tar.gz) = bbb"), + "SPECS/c/curl/sources": []byte("SHA512 (curl-1.tar.gz) = aaa"), + "README.md": []byte("unrelated change"), }, ) @@ -268,6 +331,72 @@ upstream-commit = "abcdef1234567" assert.Equal(t, []string{"curl"}, project.componentGroups["core"]) } +func TestLoadHistoricalProject_OnlyBuildsInputsForLocklessComponents(t *testing.T) { + rootConfig := []byte(` +[project] +default-distro = { name = "testdistro", version = "1.0" } +rendered-specs-dir = "SPECS" + +[distros.testdistro] +description = "Test distro" +[distros.testdistro.versions."1.0"] +release-ver = "1.0" + +[components.rust-asyncband.spec] +type = "upstream" +upstream-commit = "abcdef1234567" + +[components.curl.spec] +type = "local" +path = "unavailable/curl.spec" +`) + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{"azldev.toml": rootConfig}}, + }) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + + project, err := loadHistoricalProject(env.Env, tree, ".") + require.NoError(t, err, "locked component inputs must not be read for the lockless comparison") + assert.Contains(t, project.components, "curl") + assert.NotContains(t, project.comparisonInputs, "curl") + assert.Equal(t, "abcdef1234567", project.comparisonInputs["rust-asyncband"].SourceIdentity) +} + +func TestLoadHistoricalProject_IgnoresUnknownHistoricalFields(t *testing.T) { + rootConfig := []byte(` +[project] +default-distro = { name = "testdistro", version = "1.0" } +rendered-specs-dir = "SPECS" + +[distros.testdistro] +description = "Test distro" +[distros.testdistro.versions."1.0"] +release-ver = "1.0" + +[test-suites.legacy] +description = "Removed config table" + +[components.curl.spec] +type = "upstream" +upstream-commit = "abcdef1234567" +`) + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{"azldev.toml": rootConfig}}, + }) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + testEnv := testutils.NewTestEnvWithoutLockfile(t) + require.False(t, testEnv.Env.PermissiveConfigParsing()) + + project, err := loadHistoricalProject(testEnv.Env, tree, ".") + require.NoError(t, err) + assert.Equal(t, "abcdef1234567", project.components["curl"].Spec.UpstreamCommit) +} + func TestSelectHistoricalComponentNames_UsesBothRefs(t *testing.T) { fromProject := &historicalProject{ components: map[string]projectconfig.ComponentConfig{ diff --git a/internal/app/azldev/cmds/component/cmdoptions.go b/internal/app/azldev/cmds/component/cmdoptions.go index b347f3ca6..be0ecd424 100644 --- a/internal/app/azldev/cmds/component/cmdoptions.go +++ b/internal/app/azldev/cmds/component/cmdoptions.go @@ -4,59 +4,15 @@ package component import ( - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/spf13/cobra" ) -// cmdOptions holds the mode-sensitive choices made when a component command is -// constructed. The zero value describes azldev's default (lock file) mode. -type cmdOptions struct { - // withoutLockfile omits lock-file-specific flags, matching the command surface - // exposed by the global '--without-lockfile' flag. - withoutLockfile bool -} - -// CmdOption customizes how a component command is constructed. -type CmdOption func(*cmdOptions) - -// WithoutLockfileFlags omits the lock-file-specific flags from a component command. -// Commands are registered with this option when the global '--without-lockfile' -// flag selects lock-file-free mode, so the flags that only lock files can honor are -// never offered. -func WithoutLockfileFlags() CmdOption { - return func(options *cmdOptions) { - options.withoutLockfile = true - } -} - -// newCmdOptions resolves the supplied command options. -func newCmdOptions(opts ...CmdOption) cmdOptions { - options := cmdOptions{} - for _, opt := range opts { - opt(&options) - } - - return options -} - // addComponentFilterOptions registers the component selection flags, including the -// lock-file flags that only apply in azldev's default mode. +// lock-validation flag honored by components using lock files. func addComponentFilterOptions( - cmd *cobra.Command, filter *components.ComponentFilter, options cmdOptions, + cmd *cobra.Command, filter *components.ComponentFilter, ) { components.AddComponentFilterOptionsToCommand(cmd, filter) - - if !options.withoutLockfile { - components.AddLockValidationFlagToCommand(cmd, filter) - } -} - -// cmdOptionsForApp returns the command options matching the app's selected mode. -func cmdOptionsForApp(app *azldev.App) []CmdOption { - if app.WithoutLockfile() { - return []CmdOption{WithoutLockfileFlags()} - } - - return nil + components.AddLockValidationFlagToCommand(cmd, filter) } diff --git a/internal/app/azldev/cmds/component/component.go b/internal/app/azldev/cmds/component/component.go index dd75606fd..87024e752 100644 --- a/internal/app/azldev/cmds/component/component.go +++ b/internal/app/azldev/cmds/component/component.go @@ -31,24 +31,18 @@ components defined in the project configuration.`, buildOnAppInit(app, cmd) changedOnAppInit(app, cmd) diffSourcesOnAppInit(app, cmd) + distGitRepoOnAppInit(app, cmd) listOnAppInit(app, cmd) prepareOnAppInit(app, cmd) renderOnAppInit(app, cmd) testOnAppInit(app, cmd) metadataOnAppInit(app, cmd) - // The commands that maintain resolved component state differ by mode: the - // default mode maintains lock files, while lock-file-free mode maintains - // generated upstream-commit config. Registering only the commands that - // belong to the active mode keeps help, docs, and MCP tools honest. - if app.WithoutLockfile() { - legacyOnAppInit(app, cmd) - refreshUpstreamCommitOnAppInit(app, cmd) - } else { - historyOnAppInit(app, cmd) - queryOnAppInit(app, cmd) - updateOnAppInit(app, cmd) - } + historyOnAppInit(app, cmd) + queryOnAppInit(app, cmd) + updateOnAppInit(app, cmd) + refreshUpstreamCommitOnAppInit(app, cmd) + makeLocklessOnAppInit(app, cmd) return cmd } diff --git a/internal/app/azldev/cmds/component/component_test.go b/internal/app/azldev/cmds/component/component_test.go index 156fe6b36..73e77ab40 100644 --- a/internal/app/azldev/cmds/component/component_test.go +++ b/internal/app/azldev/cmds/component/component_test.go @@ -4,12 +4,14 @@ package component_test import ( - "slices" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx/opctx_test" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -28,51 +30,41 @@ func TestOnAppInit(t *testing.T) { assert.Contains(t, topLevelCommandNames, "component") } -func TestOnAppInit_CommandsByMode(t *testing.T) { - testCases := []struct { - name string - args []string - refreshExpected bool - }{ - {name: "default mode", args: nil, refreshExpected: false}, - {name: "lock-file-free mode", args: []string{"--without-lockfile"}, refreshExpected: true}, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - app := azldev.NewApp(opctx_test.NewMockFileSystemFactory(ctrl), opctx_test.NewMockOSEnvFactory(ctrl)) - - app.PreParseGlobalFlags(testCase.args) - component.OnAppInit(app) - - commandNames, err := app.CommandNames("component") - require.NoError(t, err) +func TestOnAppInit_RegistersBothWorkflows(t *testing.T) { + ctrl := gomock.NewController(t) + app := azldev.NewApp(opctx_test.NewMockFileSystemFactory(ctrl), opctx_test.NewMockOSEnvFactory(ctrl)) + component.OnAppInit(app) - assert.Equal(t, testCase.refreshExpected, - slices.Contains(commandNames, "refresh-upstream-commit")) + commandNames, err := app.CommandNames("component") + require.NoError(t, err) - // The lock-file command names stay reachable in lock-file-free mode, - // where they are registered as hidden no-ops for compatibility. - for _, name := range []string{"history", "query", "update"} { - assert.Contains(t, commandNames, name) - } - }) + for _, name := range []string{"history", "query", "update", "refresh-upstream-commit"} { + assert.Contains(t, commandNames, name) } } -func TestNewComponentCommands_LockValidationFlagByMode(t *testing.T) { - // The lock-file consistency flags only mean something when lock files are in - // play, so lock-file-free mode leaves them unregistered. +func TestNewComponentCommands_LockValidationFlags(t *testing.T) { assert.NotNil(t, component.NewBuildCmd().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewRenderCmd().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewComponentListCommand().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewHistoryCmd().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewComponentQueryCommand().Flags().Lookup("skip-lock-validation")) assert.NotNil(t, component.NewUpdateCmd().Flags().Lookup("skip-lock-validation")) +} - withoutLockfile := component.WithoutLockfileFlags() - assert.Nil(t, component.NewBuildCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) - assert.Nil(t, component.NewRenderCmd(withoutLockfile).Flags().Lookup("skip-lock-validation")) - assert.Nil(t, component.NewComponentListCommand(withoutLockfile).Flags().Lookup("skip-lock-validation")) +func TestLocklessComponents_SkipLegacyHistory(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{ + Name: "rust-asyncband", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + } + filter := components.ComponentFilter{IncludeAllComponents: true} + + history, err := component.ComponentHistory(env.Env, &component.HistoryOptions{ + ComponentFilter: filter, + SharedTomlMode: "show", + }) + require.NoError(t, err) + assert.Empty(t, history) } diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 7fd728050..f2e97d3f5 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -29,12 +29,12 @@ type DiffSourcesOptions struct { OutputFile string } -func diffSourcesOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewDiffSourcesCmd(cmdOptionsForApp(app)...)) +func diffSourcesOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewDiffSourcesCmd()) } // NewDiffSourcesCmd constructs a [cobra.Command] for the "component diff-sources" CLI subcommand. -func NewDiffSourcesCmd(opts ...CmdOption) *cobra.Command { +func NewDiffSourcesCmd() *cobra.Command { var options DiffSourcesOptions var cmd *cobra.Command @@ -54,7 +54,7 @@ overlays to the copy and displays the resulting diff between the two trees.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVar(&options.OutputFile, "output-file", "", "write the diff output to a file instead of stdout") @@ -102,8 +102,7 @@ func DiffComponentSources(env *azldev.Env, options *DiffSourcesOptions) (interfa } preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, - sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), - sources.WithSpecEditor(options.specEditorMode())) + diffSourcesPreparerOptions(env, component.GetName(), distro, options)...) if err != nil { return nil, fmt.Errorf("failed to create source preparer:\n%w", err) } @@ -185,3 +184,17 @@ func shouldColorize(env *azldev.Env) bool { return false } + +func diffSourcesPreparerOptions( + env *azldev.Env, componentName string, distro sourceproviders.ResolvedDistro, options *DiffSourcesOptions, +) []sources.PreparerOption { + preparerOpts := []sources.PreparerOption{ + sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), + sources.WithSpecEditor(options.specEditorMode()), + } + if env.ComponentWithoutLockfile(componentName) { + preparerOpts = append(preparerOpts, sources.WithPreservedReleaseFormatting()) + } + + return preparerOpts +} diff --git a/internal/app/azldev/cmds/component/distgitrepo.go b/internal/app/azldev/cmds/component/distgitrepo.go new file mode 100644 index 000000000..9d16753b3 --- /dev/null +++ b/internal/app/azldev/cmds/component/distgitrepo.go @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" + "github.com/spf13/afero" + "github.com/spf13/cobra" +) + +type DistGitRepoOptions struct { + Component string + OutputDir string + Force bool +} + +type DistGitRepoResult struct { + Component string `json:"component"` + OutputDir string `json:"outputDir"` +} + +func distGitRepoOnAppInit(_ *azldev.App, parent *cobra.Command) { + parent.AddCommand(NewDistGitRepoCmd()) +} + +// NewDistGitRepoCmd constructs the "component dist-git-repo" command. +func NewDistGitRepoCmd() *cobra.Command { + var options DistGitRepoOptions + + cmd := &cobra.Command{ + Use: "dist-git-repo ", + Short: "Extract a component's committed dist-git history into a repository", + Long: `Clone the local project repository's current branch with '--single-branch' +and '--no-local', then run 'git filter-repo --subdirectory-filter' to retain +only //. +The component's dist-git files become the new repository's root. + +Requires exactly one component name, '--output-dir' (-o), and git-filter-repo. +Only committed files and history are exported; this does not render the component +or clone its upstream distro repository. + +An existing output directory is rejected unless '--force' is supplied. +With '--force', that directory is deleted and recreated after preflight checks. +For safety, the source repository and home directory (including their ancestors), +Git metadata, and paths containing symbolic links cannot be used as output directories. +Clone or filtering failures leave any partial output available for inspection.`, + Example: ` azldev comp dist-git-repo curl -o ./dist-git/curl + azldev comp dist-git-repo rust-asyncband --output-dir ./dist-git/rust-asyncband --force`, + Args: cobra.ExactArgs(1), + RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { + options.Component = args[0] + + return CreateComponentDistGitRepo(env, &options) + }), + ValidArgsFunction: components.GenerateComponentNameCompletions, + } + cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", + "destination for the extracted repository (required)") + _ = cmd.MarkFlagRequired("output-dir") + _ = cmd.MarkFlagDirname("output-dir") + cmd.Flags().BoolVar(&options.Force, "force", false, "delete and recreate an existing output directory") + + return cmd +} + +// CreateComponentDistGitRepo exports a component's committed rendered subtree and its history. +func CreateComponentDistGitRepo(env *azldev.Env, options *DistGitRepoOptions) (*DistGitRepoResult, error) { + if options.OutputDir == "" { + return nil, errors.New("'--output-dir' (-o) is required") + } + + if err := fileutils.ValidateFilename(options.Component); err != nil { + return nil, fmt.Errorf("invalid component name:\n%w", err) + } + + resolver := components.NewResolver(env) + + resolver.SuppressLockWarnings = true + if _, err := resolver.GetComponentByName(options.Component); err != nil { + return nil, fmt.Errorf("resolving component:\n%w", err) + } + + output, err := distGitRepoOutputPath(env, options.OutputDir) + if err != nil { + return nil, err + } + + exists, err := fileutils.Exists(env.FS(), output) + if err != nil { + return nil, fmt.Errorf("checking output directory %#q:\n%w", output, err) + } + + if exists && !options.Force { + return nil, fmt.Errorf("output directory %#q already exists; use '--force' to replace it", output) + } + + result := &DistGitRepoResult{Component: options.Component, OutputDir: output} + if env.DryRun() { + slog.Info("Dry run; would extract component dist-git repository", "component", options.Component, "outputDir", output) + + return result, nil + } + + if err := requireGitFilterRepo(env); err != nil { + return nil, err + } + + repoRoot, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--show-toplevel") + if err != nil { + return nil, fmt.Errorf("finding project repository root:\n%w", err) + } + + if err := validateDistGitRepoOutput(env, repoRoot, output); err != nil { + return nil, err + } + + subdir, err := distGitRepoSubdirectory(env, repoRoot, options.Component) + if err != nil { + return nil, err + } + + return createDistGitRepo(env, repoRoot, subdir, output, exists, result) +} + +func requireGitFilterRepo(env *azldev.Env) error { + prereq := &prereqs.PackagePrereq{VersionCheck: func(ctx opctx.Ctx) error { + if _, err := git.RunInDir(ctx, ctx, env.ProjectDir(), "filter-repo", "--version"); err != nil { + return fmt.Errorf("git-filter-repo must be installed to use the dist-git-repo command:\n%w", err) + } + + return nil + }} + if err := prereqs.RequireExecutable(env, "git", prereq); err != nil { + return fmt.Errorf("checking dist-git-repo prerequisites:\n%w", err) + } + + return nil +} + +func distGitRepoOutputPath(env *azldev.Env, output string) (string, error) { + if !filepath.IsAbs(output) { + cwd, err := env.OSEnv().Getwd() + if err != nil { + return "", fmt.Errorf("getting current directory:\n%w", err) + } + + output = filepath.Join(cwd, output) + } + + return filepath.Clean(output), nil +} + +func validateDistGitRepoOutput(env *azldev.Env, repoRoot, output string) error { + if _, err := repoRelPath(output, repoRoot); err == nil { + return fmt.Errorf("output directory %#q must not contain the source repository", output) + } + + if home := env.OSEnv().Getenv("HOME"); home != "" { + if _, err := repoRelPath(output, home); err == nil { + return fmt.Errorf("output directory %#q must not contain the home directory", output) + } + } + + if _, err := repoRelPath(filepath.Join(repoRoot, ".git"), output); err == nil { + return fmt.Errorf("output directory %#q must not be inside source Git metadata", output) + } + + gitDir, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--git-common-dir") + if err != nil { + return fmt.Errorf("finding source Git metadata:\n%w", err) + } + + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(env.ProjectDir(), gitDir) + } + + if _, err := repoRelPath(output, gitDir); err == nil { + return fmt.Errorf("output directory %#q must not contain source Git metadata", output) + } + + if _, err := repoRelPath(gitDir, output); err == nil { + return fmt.Errorf("output directory %#q must not be inside source Git metadata", output) + } + + return validateDistGitRepoOutputLinks(env, output) +} + +func validateDistGitRepoOutputLinks(env *azldev.Env, output string) error { + lstater, ok := env.FS().(afero.Lstater) + if !ok { + return errors.New("cannot check output directory for symbolic links") + } + + for path := output; ; path = filepath.Dir(path) { + info, _, err := lstater.LstatIfPossible(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checking output path %#q:\n%w", path, err) + } + + if err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("output path %#q must not contain symbolic links", path) + } + + if path == filepath.Dir(path) { + break + } + } + + return nil +} + +func distGitRepoSubdirectory(env *azldev.Env, repoRoot, name string) (string, error) { + specDir := env.Config().Project.RenderedSpecsDir + if specDir == "" { + specDir = projectconfig.DefaultRenderedSpecsDir + } + + subdir, err := projectSpecRepoRelPath(env, repoRoot, filepath.Join(specDir, strings.ToLower(name[:1]), name)) + if err != nil { + return "", err + } + + subdir = filepath.ToSlash(subdir) + + return subdir, nil +} + +func createDistGitRepo( + env *azldev.Env, repoRoot, subdir, output string, exists bool, result *DistGitRepoResult, +) (*DistGitRepoResult, error) { + if exists { + if err := env.FS().RemoveAll(output); err != nil { + return nil, fmt.Errorf("removing output directory %#q:\n%w", output, err) + } + } + + if err := fileutils.MkdirAll(env.FS(), filepath.Dir(output)); err != nil { + return nil, fmt.Errorf("creating output parent directory:\n%w", err) + } + + if _, err := git.RunInDir(env, env, repoRoot, + "clone", "--single-branch", "--no-local", "--", repoRoot, output); err != nil { + return nil, fmt.Errorf("cloning project repository:\n%w", err) + } + + if _, err := git.RunInDir(env, env, output, "filter-repo", "--subdirectory-filter", subdir); err != nil { + return nil, fmt.Errorf("filtering component dist-git history:\n%w", err) + } + + return result, nil +} diff --git a/internal/app/azldev/cmds/component/distgitrepo_test.go b/internal/app/azldev/cmds/component/distgitrepo_test.go new file mode 100644 index 000000000..ffb3212cb --- /dev/null +++ b/internal/app/azldev/cmds/component/distgitrepo_test.go @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "errors" + "os/exec" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + distGitTestOutput = "/exports/component" + distGitTestProject = "/project" + distGitTestRevParse = "rev-parse" + distGitTestCommonDir = "--git-common-dir" + distGitTestClone = "clone" + distGitTestFilterRepo = "filter-repo" + distGitTestMetadata = "/project/.git" +) + +func newDistGitRepoTestEnv(t *testing.T, name string) *testutils.TestEnv { + t.Helper() + + env := testutils.NewTestEnv(t) + env.Config.Components[name] = projectconfig.ComponentConfig{Name: name} + env.Config.Project.RenderedSpecsDir = "/project/specs" + env.CmdFactory.RegisterCommandInSearchPath("git") + + return env +} + +func TestDistGitRepoCmd_Arguments(t *testing.T) { + for _, args := range [][]string{ + {}, + {"curl", "vim", "-o", distGitTestOutput}, + {"curl"}, + {"curl", "-o", ""}, + {"unknown-component", "-o", distGitTestOutput}, + {"cu*", "-o", distGitTestOutput}, + } { + t.Run(filepath.Join(args...), func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + cmd := componentcmds.NewDistGitRepoCmd() + cmd.SetArgs(args) + require.Error(t, cmd.ExecuteContext(env.Env)) + }) + } +} + +func TestCreateComponentDistGitRepo(t *testing.T) { + for _, testCase := range []struct { + name, component, specDir, wantSubdir string + force bool + }{ + {name: "default path", component: "curl", wantSubdir: "specs/c/curl"}, + {name: "custom relative path", component: "curl", specDir: "build/rendered", wantSubdir: "build/rendered/c/curl"}, + {name: "uppercase component", component: "SLOF", specDir: "/project/rendered", wantSubdir: "rendered/s/SLOF"}, + {name: "replace existing directory", component: "curl", wantSubdir: "specs/c/curl", force: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, testCase.component) + + env.Config.Project.RenderedSpecsDir = testCase.specDir + if testCase.force { + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + require.NoError(t, fileutils.WriteFile(env.TestFS, distGitTestOutput+"/stale", []byte("keep"), + fileperms.PublicFile)) + } + + var commands [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args) + switch cmd.Args[3] { + case distGitTestRevParse: + if cmd.Args[4] == distGitTestCommonDir { + return distGitTestMetadata, nil + } + + return distGitTestProject, nil + case distGitTestClone: + exists, err := fileutils.Exists(env.TestFS, distGitTestOutput) + require.NoError(t, err) + assert.False(t, exists, "clone must receive a fresh destination") + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + } + + return "", nil + } + result, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: testCase.component, OutputDir: distGitTestOutput, Force: testCase.force, + }) + require.NoError(t, err) + assert.Equal(t, testCase.component, result.Component) + assert.Equal(t, distGitTestOutput, result.OutputDir) + assert.Equal(t, [][]string{ + {"git", "-C", distGitTestProject, distGitTestFilterRepo, "--version"}, + {"git", "-C", distGitTestProject, distGitTestRevParse, "--show-toplevel"}, + {"git", "-C", distGitTestProject, distGitTestRevParse, distGitTestCommonDir}, + { + "git", "-C", distGitTestProject, distGitTestClone, "--single-branch", "--no-local", "--", + distGitTestProject, distGitTestOutput, + }, + {"git", "-C", distGitTestOutput, distGitTestFilterRepo, "--subdirectory-filter", testCase.wantSubdir}, + }, commands) + }) + } +} + +func TestCreateComponentDistGitRepo_ExistingOutput(t *testing.T) { + for _, nonempty := range []bool{false, true} { + t.Run(map[bool]string{false: "empty", true: "nonempty"}[nonempty], func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + + if nonempty { + require.NoError(t, fileutils.WriteFile(env.TestFS, distGitTestOutput+"/keep", []byte("keep"), + fileperms.PublicFile)) + } + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, + }) + require.ErrorContains(t, err, "already exists") + require.ErrorContains(t, err, "--force") + exists, err := fileutils.DirExists(env.TestFS, distGitTestOutput) + require.NoError(t, err) + assert.True(t, exists) + }) + } +} + +func TestCreateComponentDistGitRepo_FailuresPreserveOutput(t *testing.T) { + for _, step := range []string{distGitTestFilterRepo, distGitTestRevParse} { + t.Run(step, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, distGitTestOutput)) + + const sentinel = distGitTestOutput + "/keep" + require.NoError(t, fileutils.WriteFile(env.TestFS, sentinel, []byte("original"), fileperms.PublicFile)) + + failure := errors.New("preflight failed") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.NotEqual(t, distGitTestClone, cmd.Args[3]) + + if cmd.Args[3] == step { + return "", failure + } + + return distGitTestProject, nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, Force: true, + }) + require.ErrorIs(t, err, failure) + + if step == distGitTestFilterRepo { + require.ErrorContains(t, err, "git-filter-repo must be installed to use the dist-git-repo command") + } + + content, err := fileutils.ReadFile(env.TestFS, sentinel) + require.NoError(t, err) + assert.Equal(t, "original", string(content)) + }) + } +} + +func TestCreateComponentDistGitRepo_UnsafeOutput(t *testing.T) { + for _, output := range []string{ + "/", "/project", "/project/..", "/project/.git", "/project/.git/objects", + "/shared", "/shared/git", "/shared/git/objects", + } { + t.Run(output, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.NotEqual(t, distGitTestClone, cmd.Args[3]) + + if cmd.Args[3] == distGitTestRevParse { + if cmd.Args[4] == distGitTestCommonDir { + return "/shared/git", nil + } + + return distGitTestProject, nil + } + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: output, Force: true, + }) + require.ErrorContains(t, err, "output directory") + exists, err := fileutils.Exists(env.TestFS, "/project/azldev.toml") + require.NoError(t, err) + assert.True(t, exists, "source project must remain intact") + }) + } +} + +func TestCreateComponentDistGitRepo_CommandFailures(t *testing.T) { + for _, step := range []string{distGitTestClone, "filter"} { + t.Run(step, func(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + failure := errors.New("export failed") + + var cloned bool + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + switch cmd.Args[3] { + case distGitTestRevParse: + if cmd.Args[4] == distGitTestCommonDir { + return "/project/.git", nil + } + + return distGitTestProject, nil + case distGitTestClone: + if step == distGitTestClone { + return "", failure + } + + cloned = true + case distGitTestFilterRepo: + if cmd.Args[4] == "--subdirectory-filter" { + assert.True(t, cloned) + + return "", failure + } + } + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(env.Env, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, + }) + require.ErrorIs(t, err, failure) + }) + } +} + +func TestCreateComponentDistGitRepo_DryRun(t *testing.T) { + env := newDistGitRepoTestEnv(t, "curl") + dryEnv := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: "/project", Config: env.Config, Interfaces: env.TestInterfaces, + EventListener: env.EventListener, DryRunnable: azldev.NewAppDryRunnable(true), + }) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + _, err := componentcmds.CreateComponentDistGitRepo(dryEnv, &componentcmds.DistGitRepoOptions{ + Component: "curl", OutputDir: distGitTestOutput, + }) + require.NoError(t, err) + exists, err := fileutils.Exists(env.TestFS, distGitTestOutput) + require.NoError(t, err) + assert.False(t, exists) +} diff --git a/internal/app/azldev/cmds/component/history.go b/internal/app/azldev/cmds/component/history.go index 739d1c2c1..d8b87bf6c 100644 --- a/internal/app/azldev/cmds/component/history.go +++ b/internal/app/azldev/cmds/component/history.go @@ -63,6 +63,8 @@ func NewHistoryCmd() *cobra.Command { Use this to find which packages get the most attention (for documentation, review prioritization, or refactoring planning). +Components using the lockless workflow via lockless-components.txt are skipped. + When a component shares its source TOML with other components (e.g., a bare entry in a shared components.toml), the toml-commit count is coarse and the component is marked 'toml-shared'. Use --shared=omit to drop those rows. @@ -108,7 +110,7 @@ hand-picking entries to document.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVar(&options.SharedTomlMode, "shared", sharedTomlModeShow, "How to report rows for components that share a TOML file with others: "+ @@ -295,6 +297,11 @@ func ComponentHistory(env *azldev.Env, options *HistoryOptions) ([]HistoryResult return nil, fmt.Errorf("resolving components:\n%w", err) } + lockedComps := componentsForLockMode(env, comps.Components(), false) + if len(lockedComps) == 0 { + return []HistoryResult{}, nil + } + ctx, err := newHistoryContext(env) if err != nil { return nil, err @@ -308,7 +315,7 @@ func ComponentHistory(env *azldev.Env, options *HistoryOptions) ([]HistoryResult explicit := hasExplicitComponentSelection(&options.ComponentFilter) effectiveIncludeBare := options.IncludeBare || explicit - stubs := buildHistoryStubs(env, comps.Components(), effectiveIncludeBare) + stubs := buildHistoryStubs(env, lockedComps, effectiveIncludeBare) if len(stubs) == 0 { return nil, nil } diff --git a/internal/app/azldev/cmds/component/legacy.go b/internal/app/azldev/cmds/component/legacy.go deleted file mode 100644 index 656d9ba71..000000000 --- a/internal/app/azldev/cmds/component/legacy.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package component - -import ( - "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" - "github.com/spf13/cobra" -) - -const ( - legacyUpdateMessage = "azldev component update no longer does anything and should no longer be used." - legacyHistoryMessage = "azldev component history no longer does anything and should no longer be used." - legacyQueryMessage = "azldev component query no longer does anything and should no longer be used." -) - -func legacyOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand( - newLegacyNoOpCmd("update", nil, legacyUpdateMessage), - newLegacyNoOpCmd("history", []string{"hist"}, legacyHistoryMessage), - newLegacyNoOpCmd("query", nil, legacyQueryMessage), - ) -} - -func newLegacyNoOpCmd(name string, aliases []string, message string) *cobra.Command { - cmd := &cobra.Command{ - Use: name, - Aliases: aliases, - Short: message, - Hidden: true, - DisableFlagParsing: true, - Args: cobra.ArbitraryArgs, - Run: func(cmd *cobra.Command, _ []string) { - cmd.Println(message) - }, - } - - azldev.ExcludeFromMarkdownDocs(cmd) - - return cmd -} diff --git a/internal/app/azldev/cmds/component/legacy_internal_test.go b/internal/app/azldev/cmds/component/legacy_internal_test.go deleted file mode 100644 index 50bac09e5..000000000 --- a/internal/app/azldev/cmds/component/legacy_internal_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -package component - -import ( - "bytes" - "testing" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLegacyCommandsAreHiddenNoOps(t *testing.T) { - tests := []struct { - name string - command string - args []string - expected string - }{ - { - name: "update", - command: "update", - args: []string{"-a", "--bump", "curl"}, - expected: legacyUpdateMessage, - }, - { - name: "history", - command: "history", - args: []string{"-a", "--include-bare", "-O", "json"}, - expected: legacyHistoryMessage, - }, - { - name: "history alias", - command: "hist", - args: []string{"curl"}, - expected: legacyHistoryMessage, - }, - { - name: "query", - command: "query", - args: []string{"-p", "curl", "--arch", "aarch64", "-O", "json"}, - expected: legacyQueryMessage, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - parent := &cobra.Command{Use: "component"} - legacyOnAppInit(nil, parent) - - var output bytes.Buffer - parent.SetOut(&output) - parent.SetErr(&output) - parent.SetArgs(append([]string{test.command}, test.args...)) - - require.NoError(t, parent.Execute()) - assert.Equal(t, test.expected+"\n", output.String()) - - command, _, err := parent.Find([]string{test.command}) - require.NoError(t, err) - assert.True(t, command.Hidden) - assert.True(t, command.DisableFlagParsing) - }) - } -} diff --git a/internal/app/azldev/cmds/component/list.go b/internal/app/azldev/cmds/component/list.go index 4f0ba3ee1..271119346 100644 --- a/internal/app/azldev/cmds/component/list.go +++ b/internal/app/azldev/cmds/component/list.go @@ -19,12 +19,12 @@ type ListComponentOptions struct { ComponentFilter components.ComponentFilter } -func listOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewComponentListCommand(cmdOptionsForApp(app)...)) +func listOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewComponentListCommand()) } // Constructs a [cobra.Command] for "component list" CLI subcommand. -func NewComponentListCommand(opts ...CmdOption) *cobra.Command { +func NewComponentListCommand() *cobra.Command { options := &ListComponentOptions{} cmd := &cobra.Command{ @@ -55,14 +55,11 @@ Component name patterns support glob syntax (*, ?, []).`, azldev.ExportAsReadOnlyMCPTool(cmd) - cmdOptions := newCmdOptions(opts...) - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions) + addComponentFilterOptions(cmd, &options.ComponentFilter) // List always skips lock validation (read-only), so the flag is // meaningless here. Hide it to avoid confusion. - if !cmdOptions.withoutLockfile { - _ = cmd.Flags().MarkHidden("skip-lock-validation") - } + _ = cmd.Flags().MarkHidden("skip-lock-validation") return cmd } diff --git a/internal/app/azldev/cmds/component/lockmodes.go b/internal/app/azldev/cmds/component/lockmodes.go new file mode 100644 index 000000000..7da2dfde6 --- /dev/null +++ b/internal/app/azldev/cmds/component/lockmodes.go @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "log/slog" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" +) + +func componentsForLockMode( + env *azldev.Env, comps []components.Component, lockless bool, +) []components.Component { + selected := make([]components.Component, 0, len(comps)) + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) == lockless { + selected = append(selected, comp) + } + } + + if skipped := len(comps) - len(selected); skipped > 0 { + slog.Info("Skipping components using the other lock mode", "count", skipped) + } + + return selected +} diff --git a/internal/app/azldev/cmds/component/makelockless.go b/internal/app/azldev/cmds/component/makelockless.go new file mode 100644 index 000000000..44e29e563 --- /dev/null +++ b/internal/app/azldev/cmds/component/makelockless.go @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "log/slog" + "maps" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/cobra" +) + +func makeLocklessOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewMakeLocklessCmd()) +} + +// NewMakeLocklessCmd constructs the "component make-lockless" command. +func NewMakeLocklessCmd() *cobra.Command { + filter := &components.ComponentFilter{} + cmd := &cobra.Command{ + Use: "make-lockless", + Short: "Select the lockless workflow and refresh upstream commits", + Long: `Add selected component names to lockless-components.txt beside the top-level +azldev.toml, then run the refresh-upstream-commit pipeline for those components. + +Names are written in lexical order, one exact component name per line, +without blank lines or duplicates. Existing names are retained; selection +patterns are expanded to exact component names. +Existing lock files are left untouched. Already-listed components are refreshed +again without duplicating their entries. + +Upstream components get generated commit TOML under base/upstream-commits/. +Include that directory in azldev.toml before component-specific configuration. +Non-upstream components are listed too, but do not contact upstream providers. +The refresh pipeline removes their generated commit TOML if one exists. + +If refreshing fails, the updated selection remains in place and successful +component pins are still saved. Fix the failure and rerun this command.`, + Example: ` azldev comp make-lockless -p curl + azldev comp make-lockless curl rust-asyncband + azldev comp make-lockless -g core + azldev comp make-lockless -a`, + RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { + filter.ComponentNamePatterns = append(args, filter.ComponentNamePatterns...) + + return MakeComponentsLockless(env, filter) + }), + ValidArgsFunction: components.GenerateComponentNameCompletions, + Annotations: map[string]string{ + azldev.CommandAnnotationPermissiveConfig: "true", + }, + } + components.AddComponentFilterOptionsToCommand(cmd, filter) + + return cmd +} + +// MakeComponentsLockless selects the lockless workflow and refreshes selected component pins. +func MakeComponentsLockless( + env *azldev.Env, filter *components.ComponentFilter, +) ([]RefreshUpstreamCommitResult, error) { + if env.ProjectDir() == "" { + return nil, errors.New("no project directory configured; cannot select lockless components") + } + + filter.SkipLockValidation = true + resolver := components.NewResolver(env) + resolver.SuppressLockWarnings = true + + resolved, err := resolver.FindComponents(filter) + if err != nil { + return nil, fmt.Errorf("resolving components:\n%w", err) + } + + if resolved.Len() == 0 { + if filter.IncludeAllComponents { + return nil, nil + } + + return nil, errors.New("no components matched the filter") + } + + names := slices.Clone(resolved.Names()) + slices.Sort(names) + + if env.DryRun() { + slog.Info("Dry run; would select lockless components and refresh upstream commits", "components", names) + + return nil, nil + } + + if err := addLocklessComponents(env, names); err != nil { + return nil, err + } + + return RefreshUpstreamCommits(env, &RefreshUpstreamCommitOptions{ComponentFilter: *filter}) +} + +func addLocklessComponents(env *azldev.Env, names []string) error { + selected, err := projectconfig.ReadLocklessComponents(env.FS(), env.ProjectDir()) + if err != nil { + return fmt.Errorf("reading lockless component selection:\n%w", err) + } + + path := filepath.Join(env.ProjectDir(), "lockless-components.txt") + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("reading %#q:\n%w", path, err) + } + + for _, name := range names { + selected[name] = true + } + + updated := strings.Join(slices.Sorted(maps.Keys(selected)), "\n") + "\n" + + if string(content) != updated { + if err := fileutils.WriteFile(env.FS(), path, []byte(updated), fileperms.PublicFile); err != nil { + return fmt.Errorf("writing %#q:\n%w", path, err) + } + } + + env.Config().LocklessComponents = selected + + return nil +} diff --git a/internal/app/azldev/cmds/component/makelockless_test.go b/internal/app/azldev/cmds/component/makelockless_test.go new file mode 100644 index 000000000..62c703e50 --- /dev/null +++ b/internal/app/azldev/cmds/component/makelockless_test.go @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "errors" + "os/exec" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/upstreamcommit" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const makeLocklessTestPath = "/project/lockless-components.txt" + +func TestNewMakeLocklessCmd(t *testing.T) { + cmd := componentcmds.NewMakeLocklessCmd() + assert.Equal(t, "make-lockless", cmd.Use) + assert.NotNil(t, cmd.RunE) + assert.NotNil(t, cmd.Flags().Lookup("component")) + assert.NotNil(t, cmd.Flags().Lookup("all-components")) + assert.NotNil(t, cmd.Flags().Lookup("component-group")) + assert.Contains(t, cmd.Annotations, azldev.CommandAnnotationPermissiveConfig) +} + +func TestMakeLocklessCmd_NoComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + cmd := componentcmds.NewMakeLocklessCmd() + cmd.SetArgs([]string{"nonexistent-component"}) + require.ErrorContains(t, cmd.ExecuteContext(env.Env), "component not found") + exists, err := fileutils.Exists(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.False(t, exists) +} + +func TestMakeComponentsLockless_SelectsAndRefreshes(t *testing.T) { + for _, testCase := range []struct { + name, existing, want string + }{ + {name: "missing file", want: "curl\nrust-asyncband\n"}, + {name: "missing newline", existing: "rust-asyncband", want: "curl\nrust-asyncband\n"}, + {name: "existing whitespace", existing: "\t rust-asyncband \r\n\n", want: "curl\nrust-asyncband\n"}, + {name: "blank file", existing: "\n \t\r\n\n", want: "curl\nrust-asyncband\n"}, + {name: "insert before existing names", existing: "rust-asyncband\nvim\n", want: "curl\nrust-asyncband\nvim\n"}, + {name: "insert between existing names", existing: "curl\nvim\n", want: "curl\nrust-asyncband\nvim\n"}, + {name: "already selected", existing: "curl\nrust-asyncband\n", want: "curl\nrust-asyncband\n"}, + {name: "already selected unsorted", existing: "rust-asyncband\ncurl\n", want: "curl\nrust-asyncband\n"}, + { + name: "already selected duplicates", existing: "rust-asyncband\ncurl\nrust-asyncband\ncurl\n", + want: "curl\nrust-asyncband\n", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + gitCalls := setupMockGitWithCounter(env, "new-upstream-commit") + for _, name := range []string{"curl", "rust-asyncband", "vim"} { + addRefreshUpstreamComponent(env, name) + } + + lock := lockfile.New() + lock.UpstreamCommit = "old-locked-commit" + env.WriteLock(t, "curl", lock) + previousLock, err := fileutils.ReadFile(env.TestFS, "/project/locks/curl.lock") + require.NoError(t, err) + + if testCase.existing != "" { + require.NoError(t, fileutils.WriteFile(env.TestFS, makeLocklessTestPath, + []byte(testCase.existing), fileperms.PublicFile)) + } + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("vim", "unselected-pin")) + + filter := &components.ComponentFilter{ComponentNamePatterns: []string{"rust-*", "curl"}} + _, err = componentcmds.MakeComponentsLockless(env.Env, filter) + require.NoError(t, err) + assert.Positive(t, gitCalls.Load()) + + for _, name := range []string{"curl", "rust-asyncband"} { + assert.True(t, env.Env.ComponentWithoutLockfile(name)) + pin, exists, err := store.Get(name) + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "new-upstream-commit", pin) + } + + pin, exists, err := store.Get("vim") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "unselected-pin", pin) + + content, err := fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, testCase.want, string(content)) + + currentLock, err := fileutils.ReadFile(env.TestFS, "/project/locks/curl.lock") + require.NoError(t, err) + assert.Equal(t, previousLock, currentLock) + + gitCalls.Store(0) + + _, err = componentcmds.MakeComponentsLockless(env.Env, filter) + require.NoError(t, err) + assert.Positive(t, gitCalls.Load(), "already-listed components must still refresh") + + content, err = fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, testCase.want, string(content)) + }) + } +} + +func TestMakeComponentsLockless_NonUpstream(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Components["local-component"] = projectconfig.ComponentConfig{ + Name: "local-component", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal, Path: "local-component.spec"}, + } + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("local-component", "obsolete-pin")) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected upstream command: %v", cmd.Args) + + return "", nil + } + cmd := componentcmds.NewMakeLocklessCmd() + cmd.SetArgs([]string{"-p", "local-component"}) + require.NoError(t, cmd.ExecuteContext(env.Env)) + content, err := fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, "local-component\n", string(content)) + + _, exists, err := store.Get("local-component") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestMakeComponentsLockless_RefreshFailureKeepsSelection(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "curl") + setupMockGit(env, "unused") + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "", errors.New("upstream unavailable") + } + _, err := componentcmds.MakeComponentsLockless(env.Env, + &components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}) + require.Error(t, err) + content, err := fileutils.ReadFile(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.Equal(t, "curl\n", string(content)) + assert.True(t, env.Env.ComponentWithoutLockfile("curl")) +} + +func TestMakeComponentsLockless_InvalidSelectionFile(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "curl") + require.NoError(t, fileutils.MkdirAll(env.TestFS, makeLocklessTestPath)) + _, err := componentcmds.MakeComponentsLockless(env.Env, + &components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}) + require.ErrorContains(t, err, "expected a file") + assert.False(t, env.Env.ComponentWithoutLockfile("curl")) +} + +func TestMakeComponentsLockless_DryRun(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "curl") + dryEnv := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: "/project", Config: env.Config, Interfaces: env.TestInterfaces, + EventListener: env.EventListener, DryRunnable: azldev.NewAppDryRunnable(true), + }) + _, err := componentcmds.MakeComponentsLockless(dryEnv, + &components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}) + require.NoError(t, err) + exists, err := fileutils.Exists(env.TestFS, makeLocklessTestPath) + require.NoError(t, err) + assert.False(t, exists) + assert.False(t, dryEnv.ComponentWithoutLockfile("curl")) +} diff --git a/internal/app/azldev/cmds/component/mockproc.go b/internal/app/azldev/cmds/component/mockproc.go index b11d57ce4..929c07476 100644 --- a/internal/app/azldev/cmds/component/mockproc.go +++ b/internal/app/azldev/cmds/component/mockproc.go @@ -5,7 +5,7 @@ package component // Required-package presets for the shared MockProcessor. // -// Render needs rpmautospec (macro expansion), rpmdevtools (spectool), and git +// Locked render needs rpmautospec (macro expansion), rpmdevtools (spectool), and git // (required for rpmautospec to read commit history). python3-click is required // by rpmautospec but not declared as an RPM dependency. Ecosystem macro // packages (go-srpm-macros, etc.) are already present via @buildsys-build → diff --git a/internal/app/azldev/cmds/component/mockprocessor.go b/internal/app/azldev/cmds/component/mockprocessor.go index 7904c13fa..59de5e927 100644 --- a/internal/app/azldev/cmds/component/mockprocessor.go +++ b/internal/app/azldev/cmds/component/mockprocessor.go @@ -5,11 +5,13 @@ package component import ( "context" + "errors" "log/slog" "path/filepath" "time" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" ) @@ -22,6 +24,24 @@ const buildProvenanceMockSubdir = "azldev-provenance-mock" // scrubbing the mock chroot, so a hung scrub can't block command shutdown. const mockProcessorCleanupTimeout = 5 * time.Minute +func renderMockProcessor(env *azldev.Env, comps []components.Component) (*sources.MockProcessor, error) { + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) { + continue + } + + processor := createMockProcessor(env) + if processor == nil { + return nil, errors.New( + "mock config required for locked rendering; ensure the project has a valid distro with mock config") + } + + return processor, nil + } + + return nil, nil //nolint:nilnil // All-lockless renders do not need a mock processor. +} + // createMockProcessor creates a lazily initialized [sources.MockProcessor] // using the project/build distro's mock config. Source providers may use a // different per-component upstream distro, which must not select the build diff --git a/internal/app/azldev/cmds/component/preparerctx.go b/internal/app/azldev/cmds/component/preparerctx.go index f98f7b671..10fdab680 100644 --- a/internal/app/azldev/cmds/component/preparerctx.go +++ b/internal/app/azldev/cmds/component/preparerctx.go @@ -17,9 +17,15 @@ import ( // is derived from the component's generated upstream-commit TOML, which has no // fingerprint to compare against, so only committed changes are represented. func gitRepoPreparerOptions( - env *azldev.Env, distro sourceproviders.ResolvedDistro, + env *azldev.Env, distro sourceproviders.ResolvedDistro, componentName string, ) []sources.PreparerOption { - if env.WithoutLockfile() { + return gitRepoPreparerOptionsForMode(env, distro, env.ComponentWithoutLockfile(componentName)) +} + +func gitRepoPreparerOptionsForMode( + env *azldev.Env, distro sourceproviders.ResolvedDistro, withoutLockfile bool, +) []sources.PreparerOption { + if withoutLockfile { return []sources.PreparerOption{ sources.WithGitRepo(env, nil, ""), sources.WithoutLockfileHistory(), diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 317b7fb38..8fc598319 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -29,11 +29,11 @@ type PrepareSourcesOptions struct { SkipSources bool } -func prepareOnAppInit(app *azldev.App, sourceCmd *cobra.Command) { - sourceCmd.AddCommand(NewPrepareSourcesCmd(cmdOptionsForApp(app)...)) +func prepareOnAppInit(_ *azldev.App, sourceCmd *cobra.Command) { + sourceCmd.AddCommand(NewPrepareSourcesCmd()) } -func NewPrepareSourcesCmd(opts ...CmdOption) *cobra.Command { +func NewPrepareSourcesCmd() *cobra.Command { var options PrepareSourcesOptions var cmd *cobra.Command @@ -49,7 +49,21 @@ The result is a directory containing the spec file and all sources, ready for inspection or manual building. This is useful for verifying that overlays apply cleanly before running a full build. -Only one component may be selected at a time.`, +Only one component may be selected at a time. +For lockless components, render-style preparation writes directly to '--output-dir' +and preserves fetched upstream Git metadata. Nonempty output requires '--force'. +Current TOML and file inputs use the working tree, including modified and untracked +files. Release/changelog baselines use only committed component dist-git files; +uncommitted edits in that existing dist-git directory are ignored. +It downloads/generates source payloads before inline macro definitions and configured +overlays. Lockless specs contain %define entries, not a separate macros file, +load directive, or Source9999 entry. Archive overlays run before spec and loose-file overlays. It does not +compare rendered output, apply a final render release bump, or commit changes. +Render history comes from 'project.rendered-specs-dir'//, +independently of '--output-dir'. No synthetic history is generated. +'--without-git' removes fetched Git metadata after successful preparation. +It honors '--skip-overlays', '--skip-sources', and '--allow-no-hashes'. +Use 'azldev component dist-git-repo' to extract committed rendered history.`, Example: ` # Prepare sources for a component azldev component prep-sources -p curl -o ./build/work/scratch/curl --force @@ -67,7 +81,7 @@ Only one component may be selected at a time.`, }, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", "output directory") _ = cmd.MarkFlagRequired("output-dir") @@ -75,7 +89,7 @@ Only one component may be selected at a time.`, cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, "skip applying overlays to prepared sources") cmd.Flags().BoolVar(&options.WithoutGitRepo, "without-git", false, - "Disable dist-git repository creation (enabled by default)") + "Exclude Git metadata from prepared output (locked components also skip synthetic dist-git creation)") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) cmd.Flags().BoolVar(&options.Force, "force", false, "delete and recreate the output directory if it already exists") @@ -89,10 +103,6 @@ Only one component may be selected at a time.`, } func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) error { - if err := validatePrepareSourcesOptions(options); err != nil { - return err - } - var comps *components.ComponentSet resolver := components.NewResolver(env) @@ -114,6 +124,15 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er component := comps.Components()[0] + withoutLockfile := env.ComponentWithoutLockfile(component.GetName()) + if err := validatePrepareSourcesOptions(options, withoutLockfile); err != nil { + return err + } + + if withoutLockfile { + return prepareLocklessSources(env, component, options) + } + event := env.StartEvent("Preparing sources", "component", component.GetName(), "outputDir", options.OutputDir) defer event.End() @@ -141,7 +160,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er "synthetic history requires overlays to be applied") } - preparerOpts := buildPreparerOptions(env, distro, options) + preparerOpts := buildPreparerOptions(env, distro, options, component.GetName()) preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( preparerOpts, @@ -165,11 +184,12 @@ func buildPreparerOptions( env *azldev.Env, distro sourceproviders.ResolvedDistro, options *PrepareSourcesOptions, + componentName string, ) []sources.PreparerOption { var opts []sources.PreparerOption if !options.WithoutGitRepo && !options.SkipOverlays { - opts = append(opts, gitRepoPreparerOptions(env, distro)...) + opts = append(opts, gitRepoPreparerOptions(env, distro, componentName)...) } if options.RPMDevBumpspec { @@ -190,12 +210,12 @@ func buildPreparerOptions( return opts } -func validatePrepareSourcesOptions(options *PrepareSourcesOptions) error { +func validatePrepareSourcesOptions(options *PrepareSourcesOptions, withoutLockfile bool) error { if !options.RPMDevBumpspec { return nil } - if options.WithoutGitRepo { + if options.WithoutGitRepo && !withoutLockfile { return errors.New("'--rpmdev-bumpspec' cannot be used with '--without-git'") } diff --git a/internal/app/azldev/cmds/component/preparesources_lockless.go b/internal/app/azldev/cmds/component/preparesources_lockless.go new file mode 100644 index 000000000..b8f9cd897 --- /dev/null +++ b/internal/app/azldev/cmds/component/preparesources_lockless.go @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "fmt" + "log/slog" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// prepareLocklessSources prepares render-style sources directly in the requested output directory. +func prepareLocklessSources( + env *azldev.Env, component components.Component, options *PrepareSourcesOptions, +) error { + if err := validateOutputDir(options.OutputDir); err != nil { + return err + } + + if err := fileutils.ValidateFilename(component.GetName()); err != nil { + return fmt.Errorf("invalid component name for source preparation:\n%w", err) + } + + event := env.StartEvent("Preparing lockless sources", "component", component.GetName(), "outputDir", options.OutputDir) + defer event.End() + + distro, err := sourceproviders.ResolveDistro(env, component) + if err != nil { + return fmt.Errorf("resolving distro for %#q:\n%w", component.GetName(), err) + } + + sourceManager, err := sourceproviders.NewSourceManager(env, distro) + if err != nil { + return fmt.Errorf("creating source manager for %#q:\n%w", component.GetName(), err) + } + + if env.DryRun() { + // [CheckOutputDir] only mutates the filesystem when '--force' is set. + if !options.Force { + if err := CheckOutputDir(env, options); err != nil { + return err + } + } + + slog.Info("Dry run; would prepare lockless sources", + "component", component.GetName(), "outputDir", options.OutputDir) + + return nil + } + + if err := CheckOutputDir(env, options); err != nil { + return err + } + + preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, + locklessSourcePreparerOptions(env, component, distro, options)...) + if err != nil { + return fmt.Errorf("creating source preparer for %#q:\n%w", component.GetName(), err) + } + + if err := preparer.PrepareSources(env, component, options.OutputDir, !options.SkipOverlays); err != nil { + return fmt.Errorf("preparing sources for %#q:\n%w", component.GetName(), err) + } + + if _, err := findSpecFile(env.FS(), options.OutputDir, component.GetName()); err != nil { + return fmt.Errorf("finding prepared spec:\n%w", err) + } + + if options.WithoutGitRepo { + if err := env.FS().RemoveAll(filepath.Join(options.OutputDir, ".git")); err != nil { + return fmt.Errorf("removing prepared Git metadata:\n%w", err) + } + } + + return nil +} + +func locklessSourcePreparerOptions( + env *azldev.Env, component components.Component, distro sourceproviders.ResolvedDistro, options *PrepareSourcesOptions, +) []sources.PreparerOption { + preparerOpts := append(gitRepoPreparerOptionsForMode(env, distro, true), + sources.WithSpecEditor(options.specEditorMode()), + sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), + sources.WithoutSyntheticHistory(), + sources.WithSkipLocalSpecConfig(), + sources.WithAutoreleaseResolver(hostRenderProcessor{env: env}), + sources.WithFetchedSources(func(_ context.Context, fetched components.Component, dir string) error { + //nolint:contextcheck // The callback context is the captured preparation env. + return prepareFetchedLocklessSources(env, fetched, dir, options.specEditorMode()) + }), + ) + if options.SkipSources { + preparerOpts = append(preparerOpts, sources.WithSkipLookaside()) + } + + if options.AllowNoHashes { + preparerOpts = append(preparerOpts, sources.WithAllowNoHashes()) + } + + if options.RPMDevBumpspec && component.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeLocal { + preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) + } + + return preparerOpts +} + +func prepareFetchedLocklessSources( + env *azldev.Env, component components.Component, dir string, editor spec.EditorMode, +) error { + repo, root, err := openRenderRepo(env) + if err != nil { + return err + } + + path, err := distGitRepoSubdirectory(env, root, component.GetName()) + if err != nil { + return fmt.Errorf("locating committed render history:\n%w", err) + } + + if component.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeLocal { + return nil + } + + history, err := readLastRender(env, repo, root, path, component.GetName(), true) + if err != nil { + return fmt.Errorf("reading committed render state:\n%w", err) + } + + _, opened, err := openLocklessSpec(env, dir, component.GetName(), editor) + if err != nil { + return err + } + + version, err := opened.GetLastTag("", "Version") + if err != nil { + return fmt.Errorf("reading upstream Version:\n%w", err) + } + + update := &locklessRenderUpdate{repoRoot: root, path: path} + if err := collectRenderUpstreamChanges(env, dir, component.GetName(), version, history, update); err != nil { + return err + } + + if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, + dir, component.GetName(), editor, history); err != nil { + return fmt.Errorf("preparing changelog and release for %#q:\n%w", component.GetName(), err) + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go new file mode 100644 index 000000000..e7a6aafad --- /dev/null +++ b/internal/app/azldev/cmds/component/preparesources_lockless_internal_test.go @@ -0,0 +1,796 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "strings" + "testing" + + gogit "github.com/go-git/go-git/v5" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestLocklessPrepareSourcesPreparesDirectlyInOutput(t *testing.T) { + const autorelease = "%autorelease" + + for _, testCase := range []struct { + name string + sourceType projectconfig.SpecSourceType + autorelease bool + failFetch bool + failPayload bool + failArchive bool + withoutGit bool + bumpspec bool + }{ + {name: "local", sourceType: projectconfig.SpecSourceTypeLocal}, + {name: "local without git", sourceType: projectconfig.SpecSourceTypeLocal, withoutGit: true}, + { + name: "without git and bumpspec", sourceType: projectconfig.SpecSourceTypeLocal, + withoutGit: true, bumpspec: true, + }, + {name: "upstream", sourceType: projectconfig.SpecSourceTypeUpstream}, + {name: "upstream autorelease", sourceType: projectconfig.SpecSourceTypeUpstream, autorelease: true}, + { + name: "upstream autorelease without git", sourceType: projectconfig.SpecSourceTypeUpstream, + autorelease: true, withoutGit: true, + }, + {name: "fetch failure", sourceType: projectconfig.SpecSourceTypeLocal, failFetch: true}, + {name: "payload failure", sourceType: projectconfig.SpecSourceTypeLocal, failPayload: true}, + { + name: "payload failure without git", sourceType: projectconfig.SpecSourceTypeLocal, + failPayload: true, withoutGit: true, + }, + {name: "archive overlay failure", sourceType: projectconfig.SpecSourceTypeLocal, failArchive: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + hash := strings.Repeat("a", 128) + + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{SourceType: testCase.sourceType}, + Build: projectconfig.ComponentBuildConfig{ + Defines: map[string]string{"prepare_fixture": "ready"}, + }, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "extra-source.tar.gz", Hash: hash, HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.invalid/source"}, + }}, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Prepared summary"}, + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInFile, + Filename: "generated-input.txt", Regex: "downloaded", Replacement: "overlaid", + }, + }, + } + if testCase.failArchive { + config := env.Config.Components["curl"] + config.SourceFiles[0].Origin = projectconfig.Origin{Type: projectconfig.OriginTypeOverlay} + config.Overlays = append(config.Overlays, projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, Archive: "extra-source.tar.gz", Filename: "remove-me.txt", + }) + env.Config.Components["curl"] = config + } + + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("project")}}}) + originalRepo, originalPreparer := openRenderRepo, newSourcePreparer + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { + openRenderRepo = originalRepo + newSourcePreparer = originalPreparer + }) + + ctrl := gomock.NewController(t) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + + var preparedDir string + + fetchErr := errors.New("injected fetch failure") + payloadErr := errors.New("injected payload failure") + + release := "7%{?dist}" + if testCase.autorelease { + release = autorelease + } + + manager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, comp components.Component, dir string, opts ...sourceproviders.FetchComponentOption) error { + preparedDir = dir + + require.Equal(t, "curl", comp.GetName()) + stageLocklessFetchedSources(t, env.FS(), dir, release, opts) + + if testCase.failFetch { + return fetchErr + } + + return nil + }) + + if !testCase.failFetch { + manager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string) error { + require.Equal(t, preparedDir, dir) + stageLocklessPreparePayloads(t, env.FS(), dir) + + if testCase.failPayload { + return payloadErr + } + + return nil + }) + } + + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + env.CmdFactory.RegisterCommandInSearchPath("rpmautospec") + + var autospecCalls []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + require.Equal(t, "rpmautospec", cmd.Args[0]) + require.True(t, testCase.autorelease) + require.Equal(t, preparedDir, cmd.Dir) + + autospecCalls = append(autospecCalls, cmd.Args[1]) + if cmd.Args[1] == "generate-changelog" { + return "* Upstream changelog\n", nil + } + + require.Equal(t, "calculate-release", cmd.Args[1]) + + return "4", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected finalization command: %v", cmd.Args) + + return nil + } + + const output = "/external/output" + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(output, "keep"), + []byte("original output"), fileperms.PublicFile)) + err := PrepareComponentSources(env.Env, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: output, Force: true, + WithoutGitRepo: testCase.withoutGit, RPMDevBumpspec: testCase.bumpspec, + }) + + require.Equal(t, output, preparedDir) + + switch { + case testCase.failFetch: + require.ErrorIs(t, err, fetchErr) + case testCase.failPayload || testCase.failArchive: + if testCase.failPayload { + require.ErrorIs(t, err, payloadErr) + } else { + require.ErrorContains(t, err, "extra-source.tar.gz") + } + + specContent, err := fileutils.ReadFile(env.FS(), filepath.Join(preparedDir, "curl.spec")) + require.NoError(t, err) + assert.NotContains(t, string(specContent), "Summary: Prepared summary") + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(preparedDir, "generated-input.txt")) + require.NoError(t, err) + assert.Equal(t, "downloaded", string(content)) + default: + require.NoError(t, err) + + wantRelease := release + if testCase.autorelease { + wantRelease = "%autorelease -b 4" + + assert.Equal(t, []string{"generate-changelog", "calculate-release"}, autospecCalls) + } + + assertLocklessPreparedSourceTree(t, env.FS(), preparedDir, wantRelease, hash) + } + + wantGit := !testCase.withoutGit || err != nil + for _, name := range []string{".git/HEAD", "unreferenced"} { + exists, err := fileutils.Exists(env.FS(), filepath.Join(preparedDir, name)) + require.NoError(t, err) + assert.Equal(t, name != ".git/HEAD" || wantGit, exists, "unexpected prepared file: %s", name) + } + + exists, err := fileutils.Exists(env.FS(), filepath.Join(output, "keep")) + require.NoError(t, err) + assert.False(t, exists, "--force must remove existing output") + + exists, err = fileutils.Exists(env.FS(), filepath.Join(preparedDir, "extra-source.tar.gz")) + require.NoError(t, err) + assert.Equal(t, !testCase.failFetch, exists) + exists, err = fileutils.Exists(env.FS(), filepath.Join(output, "c/curl")) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestLocklessPrepareSourcesRejectsNonemptyOutputWithoutForce(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + } + + const output = "/external/output" + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(output, "keep"), + []byte("original output"), fileperms.PublicFile)) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + t.Fatal("nonempty output must be rejected before source preparation") + + return nil, errors.New("unexpected source preparer construction") + } + + t.Cleanup(func() { newSourcePreparer = originalPreparer }) + + err := PrepareComponentSources(env.Env, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: output, + }) + require.ErrorContains(t, err, "--force") + content, err := fileutils.ReadFile(env.FS(), filepath.Join(output, "keep")) + require.NoError(t, err) + assert.Equal(t, "original output", string(content)) +} + +func TestLocklessPrepareSourcesDryRunPreservesForcedOutput(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + } + dryEnv := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: env.Env.ProjectDir(), Config: env.Config, Interfaces: env.TestInterfaces, + EventListener: env.EventListener, DryRunnable: azldev.NewAppDryRunnable(true), + }) + + const output = "/external/output" + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(output, "keep"), + []byte("original output"), fileperms.PublicFile)) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + t.Fatal("dry run must not prepare sources") + + return nil, errors.New("unexpected source preparer construction") + } + + t.Cleanup(func() { newSourcePreparer = originalPreparer }) + + require.NoError(t, PrepareComponentSources(dryEnv, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: output, + Force: true, + })) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(output, "keep")) + require.NoError(t, err) + assert.Equal(t, "original output", string(content)) +} + +func TestLocklessPrepareSourcesUsesWorkingInputs(t *testing.T) { + const fetchCommand = "fetch" + + for _, inputDir := range []string{"", "draft"} { + t.Run("inputs/"+inputDir, func(t *testing.T) { + env, repo, hashes := locklessWorkingInputsFixture(t, inputDir) + originalRepo, originalPreparer := openRenderRepo, newSourcePreparer + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { + openRenderRepo = originalRepo + newSourcePreparer = originalPreparer + }) + + var upstreamRanges []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + require.Equal(t, "git", cmd.Args[0]) + + switch cmd.Args[3] { + case "--literal-pathspecs": + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[1], "--", "specs/c/curl/", + }, cmd.Args) + + return hashes[0], nil + case fetchCommand: + assert.Equal(t, "+refs/heads/f40:refs/remotes/origin/f40", cmd.Args[len(cmd.Args)-1]) + + return "", nil + case "rev-parse": + assert.Equal(t, "aaaaaaa^{commit}", cmd.Args[len(cmd.Args)-1]) + + return "aaaaaaa", nil + case "log": + upstreamRanges = append(upstreamRanges, cmd.Args[len(cmd.Args)-2]) + + return "ccccccc Working upstream change\n", nil + default: + t.Fatalf("unexpected preparation command: %v", cmd.Args) + + return "", errors.New("unexpected preparation command") + } + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected finalization command: %v", cmd.Args) + + return nil + } + + history, err := readLastRender(env.Env, repo, "/project", "specs/c/curl", "curl", true) + require.NoError(t, err) + assert.Equal(t, "aaaaaaa", history.previous) + assert.Equal(t, "ccccccc", history.current) + assert.Equal(t, "f40", history.previousBranch) + assert.Equal(t, "f42", history.currentBranch) + helper, err := history.tree.File("macros.inc") + require.NoError(t, err) + content, err := helper.Contents() + require.NoError(t, err) + assert.Equal(t, "committed helper", content) + + _, err = history.tree.File("untracked.txt") + require.Error(t, err) + + manager := sourceproviders_test.NewMockSourceManager(gomock.NewController(t)) + manager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), "/external/output", gomock.Any()).DoAndReturn( + func(_ context.Context, comp components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + assert.Equal(t, "ccccccc", comp.GetConfig().Spec.UpstreamCommit) + + return fileutils.WriteFile(env.FS(), filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "1", "* Upstream changelog\n")), fileperms.PublicFile) + }) + manager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), "/external/output").Return(nil) + + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + require.NoError(t, PrepareComponentSources(env.Env, &PrepareSourcesOptions{ + ComponentFilter: components.ComponentFilter{ComponentNamePatterns: []string{"curl"}}, + OutputDir: "/external/output", + })) + assert.Equal(t, []string{"aaaaaaa..ccccccc"}, upstreamRanges) + + prepared, err := fileutils.ReadFile(env.FS(), "/external/output/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(prepared), "Release: 9\n") + assert.Contains(t, string(prepared), "* Committed changelog\n") + assert.NotContains(t, string(prepared), "* Upstream changelog\n") + + copied, err := fileutils.ReadFile(env.FS(), "/external/output/overlay-input.txt") + require.NoError(t, err) + assert.Equal(t, "working overlay input", string(copied)) + + assert.Contains(t, string(prepared), "%define working_input working\n") + + exists, err := fileutils.Exists(env.FS(), "/external/output/curl.azl.macros") + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func locklessWorkingInputsFixture( + t *testing.T, inputDir string, +) (env *testutils.TestEnv, repo *gogit.Repository, hashes []string) { + t.Helper() + + const ( + project = `[project] +default-distro = { name = "test", version = "3" } +[distros.test.versions."1"] +release-ver = "1" +dist-git-branch = "f40" +[distros.test.versions."2"] +release-ver = "2" +dist-git-branch = "f41" +[distros.test.versions."3"] +release-ver = "3" +dist-git-branch = "f42" +` + component = `[components.curl.spec] +type = 'upstream' +upstream-distro = { name = 'test', version = '1' } +[components.curl.build.defines] +working_input = 'committed' +[[components.curl.overlays]] +type = 'file-add' +file = 'overlay-input.txt' +source = 'input.txt' +` + ) + + root := "includes = [\"component.toml\", \"pins/*.toml\"]\n" + project + repo, hashes = testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{ + "azldev.toml": []byte(root), + "lockless-components.txt": []byte("curl\n"), + "component.toml": []byte(component), + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'aaaaaaa'\n"), + "input.txt": []byte("committed overlay input"), + "specs/c/curl/curl.spec": []byte(renderTestSpec("1", "9", "* Committed changelog\n")), + "specs/c/curl/macros.inc": []byte("committed helper"), + }}, + {files: map[string][]byte{ + "component.toml": []byte(strings.Replace(component, "version = '1'", "version = '2'", 1)), + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'bbbbbbb'\n"), + }}, + }) + env = testutils.NewTestEnv(t) + tree, err := resolveTree(repo, hashes[1]) + require.NoError(t, err) + require.NoError(t, copyTreeToFS(tree, env.FS(), "/project")) + + workingComponent := strings.Replace(component, "version = '1'", "version = '3'", 1) + workingComponent = strings.Replace(workingComponent, "'committed'", "'working'", 1) + working := map[string][]byte{ + "azldev.toml": []byte(fmt.Sprintf("includes = [%q, %q]\n%s", + filepath.Join(inputDir, "component.toml"), filepath.Join(inputDir, "pins/*.toml"), project)), + filepath.Join(inputDir, "component.toml"): []byte(workingComponent), + filepath.Join(inputDir, "pins/curl.toml"): []byte("[components.curl.spec]\nupstream-commit = 'ccccccc'\n"), + filepath.Join(inputDir, "input.txt"): []byte("working overlay input"), + "specs/c/curl/curl.spec": []byte("invalid uncommitted dist-git spec"), + "specs/c/curl/macros.inc": []byte("uncommitted helper"), + "specs/c/curl/untracked.txt": []byte("untracked dist-git sidecar"), + } + worktree, err := repo.Worktree() + require.NoError(t, err) + + for path, content := range working { + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join("/project", path), content, fileperms.PublicFile)) + file, err := worktree.Filesystem.Create(path) + require.NoError(t, err) + _, err = file.Write(content) + require.NoError(t, err) + require.NoError(t, file.Close()) + } + + _, config, err := projectconfig.LoadProjectConfig(env.FS(), env.Env.OSEnv(), + "/project", true, "/work", nil, false, false) + require.NoError(t, err) + + *env.Config = *config + + return env, repo, hashes +} + +func TestLocklessPrepareSourcesFlags(t *testing.T) { + const helloSHA256 = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + + for _, testCase := range []struct { + name string + options PrepareSourcesOptions + missingHash bool + missingType bool + archive bool + wantError string + }{ + {name: "skip overlays", options: PrepareSourcesOptions{SkipOverlays: true}}, + {name: "skip sources", options: PrepareSourcesOptions{SkipSources: true}, archive: true}, + {name: "skip both", options: PrepareSourcesOptions{SkipOverlays: true, SkipSources: true}}, + {name: "allow missing hash", options: PrepareSourcesOptions{AllowNoHashes: true}, missingHash: true}, + { + name: "default hash algorithm", options: PrepareSourcesOptions{AllowNoHashes: true}, + missingHash: true, missingType: true, + }, + {name: "missing hash rejected", missingHash: true, wantError: "missing required 'hash'"}, + { + name: "missing hash needs payload", options: PrepareSourcesOptions{AllowNoHashes: true, SkipSources: true}, + missingHash: true, wantError: "downloads were skipped", + }, + { + name: "skip overlays leaves missing hash unresolved", + options: PrepareSourcesOptions{AllowNoHashes: true, SkipOverlays: true}, missingHash: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ref := projectconfig.SourceFileReference{ + Filename: "extra-source.tar.gz", Hash: helloSHA256, HashType: fileutils.HashTypeSHA256, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.invalid/source"}, + } + if testCase.missingHash { + ref.Hash = "" + } + + if testCase.missingType { + ref.HashType = "" + } + + env, dir, err := runLocklessPrepareFlagFixture(t, testCase.options, ref, testCase.archive) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + exists, err := fileutils.Exists(env.FS(), filepath.Join(dir, ref.Filename)) + require.NoError(t, err) + assert.Equal(t, !testCase.options.SkipSources, exists) + exists, err = fileutils.Exists(env.FS(), filepath.Join(dir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists) + + content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, "sources")) + require.NoError(t, err) + specContent, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, "curl.spec")) + require.NoError(t, err) + + assert.NotContains(t, string(specContent), "%{load:") + assert.NotContains(t, string(specContent), "Source9999") + + if testCase.options.SkipOverlays { + assert.Equal(t, "# upstream metadata\n", string(content)) + assert.Equal(t, renderTestSpec("1", "7", ""), string(specContent)) + } else { + assert.Contains(t, string(specContent), "Summary: Prepared summary\n") + assert.Contains(t, string(specContent), "%define prepare_fixture ready\n") + + hash, hashType := helloSHA256, fileutils.HashTypeSHA256 + if testCase.missingType { + hashType = fileutils.HashTypeSHA512 + hash = "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f" + + "989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f" + } + + assert.Contains(t, string(content), string(hashType)+" ("+ref.Filename+") = "+hash) + } + }) + } +} + +func runLocklessPrepareFlagFixture( + t *testing.T, options PrepareSourcesOptions, ref projectconfig.SourceFileReference, archiveOverlay bool, +) (env *testutils.TestEnv, dir string, err error) { + t.Helper() + + env = testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": true} + + config := projectconfig.ComponentConfig{ + Name: "curl", Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + Build: projectconfig.ComponentBuildConfig{Defines: map[string]string{"prepare_fixture": "ready"}}, + SourceFiles: []projectconfig.SourceFileReference{ref}, + Overlays: []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Prepared summary", + }}, + } + if archiveOverlay { + config.Overlays = append(config.Overlays, projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayRemoveFile, Archive: "upstream.tar.gz", Filename: "vendor/**", + }) + config.SourceFiles = append(config.SourceFiles, projectconfig.SourceFileReference{ + Filename: "upstream.tar.gz", Hash: strings.Repeat("a", 64), HashType: fileutils.HashTypeSHA256, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeOverlay}, + }) + } + + env.Config.Components["curl"] = config + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("project")}}}) + originalRepo, originalPreparer := openRenderRepo, newSourcePreparer + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { + openRenderRepo = originalRepo + newSourcePreparer = originalPreparer + }) + manager := sourceproviders_test.NewMockSourceManager(gomock.NewController(t)) + manager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func( + _ context.Context, _ components.Component, fetchedDir string, opts ...sourceproviders.FetchComponentOption, + ) error { + dir = fetchedDir + + var resolved sourceproviders.FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + assert.Equal(t, options.SkipSources, resolved.SkipLookaside) + assert.True(t, resolved.PreserveGitDir) + assert.True(t, resolved.SkipLocalSpecConfig) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "7", "")), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(dir, ".git/HEAD"), + []byte("upstream history"), fileperms.PublicFile)) + + return fileutils.WriteFile(env.FS(), filepath.Join(dir, "sources"), + []byte("# upstream metadata\n"), fileperms.PublicFile) + }) + + if !options.SkipSources { + acquire := func(_ context.Context, _ components.Component, fetchedDir string) error { + exists, err := fileutils.Exists(env.FS(), filepath.Join(fetchedDir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists, "payload acquisition must precede macro-file setup") + + return fileutils.WriteFile(env.FS(), filepath.Join(fetchedDir, ref.Filename), + []byte("hello world"), fileperms.PublicFile) + } + manager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(acquire) + } + + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + opts ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, opts...) + } + options.OutputDir = "/external/output" + options.ComponentFilter = components.ComponentFilter{ComponentNamePatterns: []string{"curl"}} + err = PrepareComponentSources(env.Env, &options) + + return env, dir, err +} + +func TestLocklessPrepareSourcesUsesConfiguredRenderHistory(t *testing.T) { + for _, testCase := range []struct { + name, configuredDir, historyPath, wantError string + }{ + {name: "default", historyPath: "specs/c/Curl"}, + {name: "relative", configuredDir: "custom-specs", historyPath: "custom-specs/c/Curl"}, + {name: "absolute", configuredDir: "/project/SPECS", historyPath: "SPECS/c/Curl"}, + {name: "outside repository", configuredDir: "/external/specs", wantError: "escapes repository root"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = testCase.configuredDir + + files := map[string][]byte{"README": []byte("project")} + if testCase.historyPath != "" { + files[testCase.historyPath+"/Curl.spec"] = []byte(renderTestSpec("1", "9", "")) + } + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: files}}) + originalRepo := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, renderTestProjectRoot, nil + } + + t.Cleanup(func() { openRenderRepo = originalRepo }) + + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return("Curl") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + }) + + historyErr := errors.New("injected history lookup failure") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", renderTestProjectRoot, "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", testCase.historyPath + "/", + }, cmd.Args) + + return "", historyErr + } + + err := prepareFetchedLocklessSources(env.Env, component, "/work/staging", spec.EditorLegacy) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.ErrorIs(t, err, historyErr) + } + }) + } +} + +func assertLocklessPreparedSourceTree(t *testing.T, fs opctx.FS, dir, release, hash string) { + t.Helper() + + specContent, err := fileutils.ReadFile(fs, filepath.Join(dir, "curl.spec")) + require.NoError(t, err) + assert.Contains(t, string(specContent), "Summary: Prepared summary\n") + assert.NotContains(t, string(specContent), "%{load:") + assert.NotContains(t, string(specContent), "Source9999") + + exists, err := fileutils.Exists(fs, filepath.Join(dir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists) + assert.Contains(t, string(specContent), "Release: "+release+"\n") + + content, err := fileutils.ReadFile(fs, filepath.Join(dir, "generated-input.txt")) + require.NoError(t, err) + assert.Equal(t, "overlaid", string(content)) + + manifest, err := fileutils.ReadFile(fs, filepath.Join(dir, "sources")) + require.NoError(t, err) + assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = "+hash) +} + +func stageLocklessFetchedSources( + t *testing.T, fs opctx.FS, dir, release string, opts []sourceproviders.FetchComponentOption, +) { + t.Helper() + + var resolved sourceproviders.FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + assert.True(t, resolved.PreserveGitDir) + assert.False(t, resolved.SkipLookaside) + assert.True(t, resolved.SkipLocalSpecConfig) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", release, "%autochangelog\n")), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, ".git/HEAD"), + []byte("upstream history"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "unreferenced"), + []byte("keep sidecar"), fileperms.PublicFile)) +} + +func stageLocklessPreparePayloads(t *testing.T, fs opctx.FS, dir string) { + t.Helper() + + head, err := fileutils.ReadFile(fs, filepath.Join(dir, ".git/HEAD")) + require.NoError(t, err) + assert.Equal(t, "upstream history", string(head), "Git metadata must remain during preparation") + + specContent, err := fileutils.ReadFile(fs, filepath.Join(dir, "curl.spec")) + require.NoError(t, err) + assert.NotContains(t, string(specContent), "%{load:") + assert.NotContains(t, string(specContent), "Source9999:") + assert.NotContains(t, string(specContent), "Summary: Prepared summary") + + exists, err := fileutils.Exists(fs, filepath.Join(dir, "curl.azl.macros")) + require.NoError(t, err) + assert.False(t, exists, "payload acquisition must precede macro-file setup") + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "extra-source.tar.gz"), + []byte("source payload"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, "generated-input.txt"), + []byte("downloaded"), fileperms.PublicFile)) +} diff --git a/internal/app/azldev/cmds/component/preparesources_test.go b/internal/app/azldev/cmds/component/preparesources_test.go index b7d8bc46d..b47f57878 100644 --- a/internal/app/azldev/cmds/component/preparesources_test.go +++ b/internal/app/azldev/cmds/component/preparesources_test.go @@ -7,7 +7,9 @@ import ( "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" @@ -63,8 +65,10 @@ func TestPrepareSourcesCmd_NoMatch(t *testing.T) { require.Error(t, err) } -func TestPrepareSourcesOptions_RejectRPMDevBumpspecInvalidCombinationsBeforeDiscovery(t *testing.T) { +func TestPrepareSourcesOptions_RejectRPMDevBumpspecInvalidCombinations(t *testing.T) { testEnv := testutils.NewTestEnv(t) + testEnv.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + for _, testCase := range []struct { name string options component.PrepareSourcesOptions @@ -82,6 +86,9 @@ func TestPrepareSourcesOptions_RejectRPMDevBumpspecInvalidCombinationsBeforeDisc }, } { t.Run(testCase.name, func(t *testing.T) { + testCase.options.ComponentFilter = components.ComponentFilter{ + ComponentNamePatterns: []string{"curl"}, SkipLockValidation: true, + } err := component.PrepareComponentSources(testEnv.Env, &testCase.options) require.EqualError(t, err, testCase.message) }) diff --git a/internal/app/azldev/cmds/component/query.go b/internal/app/azldev/cmds/component/query.go index 83014b12c..7d279b4dd 100644 --- a/internal/app/azldev/cmds/component/query.go +++ b/internal/app/azldev/cmds/component/query.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "path/filepath" + "slices" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" @@ -49,11 +50,16 @@ func NewComponentQueryCommand() *cobra.Command { This command reads the post-overlay specs from the project's rendered-specs-dir (produced by 'azldev component render') and runs rpmspec against them in a -single shared mock chroot, batching all specs into one chroot invocation with -parallel per-spec processing. For each component, it reports the source NEVR +single shared mock chroot, querying specs in batches with parallel per-spec +processing. For each component, it reports the source NEVR and the list of binary subpackages the spec would produce when built. -The rendered-specs-dir must exist on disk; if it doesn't, run +For lockless components, queries use committed rendered files from a temporary +dist-git repository extracted with git-filter-repo. rpmautospec resolves the +release and changelog in that temporary repository before querying. Uncommitted +rendered changes are ignored. Requires git-filter-repo for lockless components. + +For locked components, rendered-specs-dir must exist on disk; if it doesn't, run 'azldev component render' first. Components that previously failed to render (those with a RENDER_FAILED marker file) are skipped with a warning.`, Example: ` # Query a single component @@ -69,7 +75,7 @@ The rendered-specs-dir must exist on disk; if it doesn't, run ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().Var(&options.Arch, "arch", "Target architecture passed to rpmspec via --target (x86_64, aarch64). "+ @@ -110,29 +116,24 @@ type componentDetails struct { func QueryComponents( env *azldev.Env, options *QueryComponentsOptions, ) ([]*componentDetails, error) { - renderedSpecsDir := env.Config().Project.RenderedSpecsDir - if renderedSpecsDir == "" { - return nil, errors.New( - "project.rendered-specs-dir is not configured; " + - "set it in the project config and run 'azldev component render' first") - } + resolver := components.NewResolver(env) + comps, resolveErr := resolver.FindComponents(&options.ComponentFilter) - dirExists, err := fileutils.DirExists(env.FS(), renderedSpecsDir) - if err != nil { - return nil, fmt.Errorf("checking rendered-specs-dir %#q:\n%w", renderedSpecsDir, err) - } + var lockedComps, locklessComps []components.Component - if !dirExists { - return nil, fmt.Errorf( - "rendered-specs-dir %#q does not exist; run 'azldev component render' first", - renderedSpecsDir) + if resolveErr == nil { + lockedComps, locklessComps = splitQueryComponents(env, comps.Components()) } - resolver := components.NewResolver(env) + renderedSpecsDir := env.Config().Project.RenderedSpecsDir - comps, err := resolver.FindComponents(&options.ComponentFilter) - if err != nil { - return nil, fmt.Errorf("failed to resolve components:\n%w", err) + checkRenderedDir := len(locklessComps) == 0 || len(lockedComps) > 0 + if err := validateQueryRenderedDirectory(env, renderedSpecsDir, checkRenderedDir); err != nil { + return nil, err + } + + if resolveErr != nil { + return nil, fmt.Errorf("failed to resolve components:\n%w", resolveErr) } cfg := env.Config() @@ -142,12 +143,18 @@ func QueryComponents( "please use command-line options to indicate which components to query") } - inputs, _, err := buildSpecQueryInputs(env, comps.Components(), renderedSpecsDir) + if len(locklessComps) > 0 { + if err := requireGitFilterRepo(env); err != nil { + return nil, err + } + } + + inputs, _, err := buildSpecQueryInputs(env, lockedComps, renderedSpecsDir) if err != nil { return nil, err } - if len(inputs) == 0 { + if len(inputs) == 0 && len(locklessComps) == 0 { return nil, fmt.Errorf( "none of the %d resolved components have a rendered spec on disk; "+ "run 'azldev component render' first", @@ -155,6 +162,10 @@ func QueryComponents( } mockProcessor := createQueryMockProcessor(env) + if len(locklessComps) > 0 { + mockProcessor = newMockProcessor(env, "", append(mockPackagesForQuery(), mockPackagesForRender()...)) + } + if mockProcessor == nil { return nil, errors.New( "mock config required for querying; ensure the project has a valid distro with mock config") @@ -190,6 +201,24 @@ func QueryComponents( return nil, fmt.Errorf("batch-querying rendered specs:\n%w", err) } + if len(locklessComps) > 0 { + locklessResults, queryErr := queryLocklessComponents(env, mockProcessor, locklessComps, scratchDir, archStr) + if queryErr != nil { + return nil, queryErr + } + + queryResults = append(queryResults, locklessResults...) + + order := make(map[string]int, comps.Len()) + for idx, comp := range comps.Components() { + order[comp.GetName()] = idx + } + + slices.SortStableFunc(queryResults, func(a, b sources.SpecQueryResult) int { + return order[a.Name] - order[b.Name] + }) + } + allDetails := make([]*componentDetails, 0, len(queryResults)) var ( diff --git a/internal/app/azldev/cmds/component/query_lockless.go b/internal/app/azldev/cmds/component/query_lockless.go new file mode 100644 index 000000000..f4ff11d2f --- /dev/null +++ b/internal/app/azldev/cmds/component/query_lockless.go @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "fmt" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/parmap" +) + +func splitQueryComponents( + env *azldev.Env, comps []components.Component, +) (locked, lockless []components.Component) { + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) { + lockless = append(lockless, comp) + } else { + locked = append(locked, comp) + } + } + + return locked, lockless +} + +func validateQueryRenderedDirectory(env *azldev.Env, dir string, required bool) error { + if !required { + return nil + } + + if dir == "" { + return errors.New("project.rendered-specs-dir is not configured; " + + "set it in the project config and run 'azldev component render' first") + } + + exists, err := fileutils.DirExists(env.FS(), dir) + if err != nil { + return fmt.Errorf("checking rendered-specs-dir %#q:\n%w", dir, err) + } + + if !exists { + return fmt.Errorf("rendered-specs-dir %#q does not exist; run 'azldev component render' first", dir) + } + + return nil +} + +func queryLocklessComponents( + env *azldev.Env, processor *sources.MockProcessor, comps []components.Component, scratchDir, arch string, +) ([]sources.SpecQueryResult, error) { + stagingDir := filepath.Join(scratchDir, "dist-git") + if err := extractQueryDistGitRepos(env, comps, stagingDir); err != nil { + return nil, err + } + + processInputs := make([]sources.ComponentInput, 0, len(comps)) + queryInputs := make([]sources.SpecQueryInput, 0, len(comps)) + + for _, comp := range comps { + name := comp.GetName() + specFilename := name + ".spec" + processInputs = append(processInputs, sources.ComponentInput{ + Name: name, SpecFilename: specFilename, SkipSpectool: true, + }) + cfg := comp.GetConfig() + queryInputs = append(queryInputs, sources.SpecQueryInput{ + Name: name, SpecRelPath: filepath.ToSlash(filepath.Join(name, specFilename)), + With: cfg.Build.With, Without: cfg.Build.Without, Defines: cfg.Build.Defines, + }) + } + + processed, err := processor.BatchProcess(env, env, stagingDir, processInputs, env.FS(), env.CPUBoundConcurrency()) + if err != nil { + return nil, fmt.Errorf("processing lockless dist-git specs:\n%w", err) + } + + var ( + failures []sources.SpecQueryResult + ready []sources.SpecQueryInput + ) + + for idx, result := range processed { + if result.Error != nil { + failures = append(failures, sources.SpecQueryResult{Name: result.Name, Error: result.Error}) + } else { + ready = append(ready, queryInputs[idx]) + } + } + + results, err := processor.BatchQuerySpecs(env, env, ready, sources.BatchQuerySpecsOptions{ + SpecsDir: stagingDir, ScratchDir: scratchDir, Arch: arch, FS: env.FS(), MaxWorkers: env.CPUBoundConcurrency(), + }) + if err != nil { + return nil, fmt.Errorf("querying lockless dist-git specs:\n%w", err) + } + + return append(results, failures...), nil +} + +func extractQueryDistGitRepos(env *azldev.Env, comps []components.Component, stagingDir string) error { + if env.DryRun() { + return nil + } + + repoRoot, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--show-toplevel") + if err != nil { + return fmt.Errorf("finding query project repository:\n%w", err) + } + + if err := validateDistGitRepoOutput(env, repoRoot, stagingDir); err != nil { + return err + } + + if err := fileutils.MkdirAll(env.FS(), stagingDir); err != nil { + return fmt.Errorf("creating query dist-git directory:\n%w", err) + } + + source := filepath.Join(filepath.Dir(stagingDir), "project.git") + if _, err := git.RunInDir(env, env, repoRoot, + "clone", "--single-branch", "--no-local", "--bare", "--", repoRoot, source); err != nil { + return fmt.Errorf("cloning query project snapshot:\n%w", err) + } + + workerEnv, cancel := env.WithCancel() + defer cancel() + + results := parmap.Map(workerEnv, env.CPUBoundConcurrency(), comps, nil, + func(ctx context.Context, comp components.Component) error { + return extractQueryComponent(ctx, workerEnv, repoRoot, source, stagingDir, comp) + }) + + var failures []error + + for idx, result := range results { + extractErr := result.Value + if result.Cancelled { + extractErr = workerEnv.Err() + } + + if extractErr != nil { + failures = append(failures, fmt.Errorf("extracting dist-git for %#q:\n%w", + comps[idx].GetName(), extractErr)) + } + } + + return errors.Join(failures...) +} + +func extractQueryComponent( + ctx context.Context, env *azldev.Env, repoRoot, source, stagingDir string, comp components.Component, +) error { + name := comp.GetName() + if err := fileutils.ValidateFilename(name); err != nil { + return fmt.Errorf("invalid query component name:\n%w", err) + } + + subdir, err := distGitRepoSubdirectory(env, repoRoot, name) + if err != nil { + return err + } + + output := filepath.Join(stagingDir, name) + if _, err := git.RunInDir(ctx, env, repoRoot, + "clone", "--shared", "--no-checkout", "--single-branch", "--", source, output); err != nil { + return fmt.Errorf("cloning shared query history:\n%w", err) + } + + if _, err := git.RunInDir(ctx, env, output, + "filter-repo", "--force", "--subdirectory-filter", subdir); err != nil { + return fmt.Errorf("filtering query component history:\n%w", err) + } + + // Copy only the filtered component's reachable objects before moving it into Mock. + if _, err := git.RunInDir(ctx, env, output, "repack", "-a", "-d"); err != nil { + return fmt.Errorf("packing query component history:\n%w", err) + } + + alternates := filepath.Join(output, ".git", "objects", "info", "alternates") + + exists, err := fileutils.Exists(env.FS(), alternates) + if err != nil { + return fmt.Errorf("checking shared query objects:\n%w", err) + } + + if exists { + if err := env.FS().Remove(alternates); err != nil { + return fmt.Errorf("detaching shared query objects:\n%w", err) + } + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/query_test.go b/internal/app/azldev/cmds/component/query_test.go index 8e265ffbf..531629c36 100644 --- a/internal/app/azldev/cmds/component/query_test.go +++ b/internal/app/azldev/cmds/component/query_test.go @@ -5,10 +5,12 @@ package component_test import ( "encoding/json" + "errors" "fmt" "os/exec" "path/filepath" "slices" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" @@ -85,6 +87,246 @@ func TestNewComponentQueryCommand(t *testing.T) { assert.Equal(t, "query", cmd.Use) } +type queryDistGitFixture struct { + cloneDir string + filtered bool + fullClones, sharedClones, repacked int +} + +func mockQueryDistGit( + t *testing.T, env *testutils.TestEnv, failure string, sentinel error, +) *queryDistGitFixture { + t.Helper() + + fixture := &queryDistGitFixture{} + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] != "git" { + return "", nil + } + + switch cmd.Args[3] { + case "rev-parse": + if cmd.Args[4] == "--git-common-dir" { + return distGitTestMetadata, nil + } + + return "/project", nil + case distGitTestClone: + if failure == distGitTestClone { + return "", sentinel + } + + if slices.Contains(cmd.Args, "--bare") { + fixture.fullClones++ + + assert.Equal(t, []string{"--single-branch", "--no-local", "--bare", "--"}, cmd.Args[4:8]) + + return "", nil + } + + fixture.sharedClones++ + + assert.Equal(t, []string{"--shared", "--no-checkout", "--single-branch", "--"}, cmd.Args[4:8]) + fixture.cloneDir = cmd.Args[len(cmd.Args)-1] + assert.Equal(t, filepath.Join(filepath.Dir(filepath.Dir(fixture.cloneDir)), "project.git"), + cmd.Args[len(cmd.Args)-2]) + require.NoError(t, fileutils.WriteFile(env.FS(), + filepath.Join(fixture.cloneDir, ".git", "objects", "info", "alternates"), + []byte("shared objects"), fileperms.PublicFile)) + case distGitTestFilterRepo: + if failure == distGitTestFilterRepo { + return "", sentinel + } + + if slices.Contains(cmd.Args, "--subdirectory-filter") { + if failure == "filter" { + return "", sentinel + } + + name := filepath.Base(cmd.Args[2]) + assert.Equal(t, "specs/"+strings.ToLower(name[:1])+"/"+name, cmd.Args[len(cmd.Args)-1]) + + fixture.filtered = true + + require.NoError(t, fileutils.WriteFile(env.FS(), filepath.Join(cmd.Args[2], name+".spec"), + []byte("committed spec"), fileperms.PublicFile)) + } + case "repack": + if failure == "repack" { + return "", sentinel + } + + fixture.repacked++ + + assert.Equal(t, []string{"-a", "-d"}, cmd.Args[4:]) + } + + return "", nil + } + + return fixture +} + +func TestQueryComponents_LocklessCommittedDistGit(t *testing.T) { + for _, testCase := range []struct { + name, failure string + mixed bool + multiple bool + }{ + {name: "lockless"}, + {name: "mixed", mixed: true}, + {name: "multiple lockless components", multiple: true}, + {name: "missing filter-repo", failure: distGitTestFilterRepo}, + {name: "snapshot clone failure", failure: distGitTestClone}, + {name: "filter failure", failure: "filter"}, + {name: "repack failure", failure: "repack"}, + {name: "processing failure", failure: "process"}, + {name: "query failure", failure: "query"}, + } { + t.Run(testCase.name, func(t *testing.T) { + failure := testCase.failure + env := newDistGitRepoTestEnv(t, testComponentName) + env.Env.SetConcurrency(1) + + names := []string{testComponentName} + if testCase.multiple { + names = append(names, "rust-asyncband") + env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{Name: "rust-asyncband"} + } + + env.Config.LocklessComponents = make(map[string]bool, len(names)) + for _, name := range names { + env.Config.LocklessComponents[name] = true + } + + if testCase.mixed { + env.Config.Components["bash"] = projectconfig.ComponentConfig{ + Name: "bash", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal, Path: "/project/bash.spec"}, + } + prepareSuccessfulBatchQuery(t, env, "bash", "/project/bash.spec") + } + + lockedQueryHandler := env.CmdFactory.RunHandler + env.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + sentinel := errors.New("injected failure") + + fixture := mockQueryDistGit(t, env, failure, sentinel) + + var processed, queried bool + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + args := strings.Join(cmd.Args, " ") + if !strings.Contains(args, "--chroot") { + return nil + } + + if strings.Contains(args, "render_process.py") { + if failure == "process" { + return sentinel + } + + processed = true + + assert.True(t, fixture.filtered) + + var records []map[string]any + + for _, name := range names { + dir := filepath.Join(filepath.Dir(fixture.cloneDir), name) + content, err := fileutils.ReadFile(env.FS(), filepath.Join(dir, name+".spec")) + require.NoError(t, err) + assert.Equal(t, "committed spec", string(content)) + + exists, err := fileutils.Exists(env.FS(), filepath.Join(dir, ".git", "objects", "info", "alternates")) + require.NoError(t, err) + assert.False(t, exists, "component repositories must not depend on host-only shared paths") + + records = append(records, map[string]any{"name": name, "specFiles": "", "error": nil}) + } + + data, err := json.Marshal(records) + require.NoError(t, err) + + return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(fixture.cloneDir), "results.json"), + data, fileperms.PublicFile) + } + + if strings.Contains(args, "query_process.py") { + if testCase.mixed && !fixture.filtered { + return lockedQueryHandler(cmd) + } + + if failure == "query" { + return sentinel + } + + queried = true + + assert.True(t, processed) + + var records []map[string]any + for _, name := range names { + records = append(records, map[string]any{ + "name": name, "srpmOut": "name=" + name + "\nepoch=0\nversion=1\nrelease=3\n", + "binOut": "", "error": nil, + }) + } + + data, err := json.Marshal(records) + require.NoError(t, err) + + return fileutils.WriteFile(env.FS(), filepath.Join(filepath.Dir(filepath.Dir(fixture.cloneDir)), "results.json"), + data, fileperms.PublicFile) + } + + return nil + } + + filter := components.ComponentFilter{ComponentNamePatterns: names} + if testCase.mixed { + filter.IncludeAllComponents = true + } + + results, queryErr := component.QueryComponents(env.Env, &component.QueryComponentsOptions{ + ComponentFilter: filter, + }) + dirs, err := fileutils.Glob(env.FS(), filepath.Join(env.Env.WorkDir(), "azldev-query-scratch-*")) + require.NoError(t, err) + assert.Empty(t, dirs, "temporary clones must be removed after success or failure") + + if failure != "" { + require.ErrorIs(t, queryErr, sentinel) + + if failure == distGitTestFilterRepo { + assert.Contains(t, queryErr.Error(), "must be installed") + } + + return + } + + require.NoError(t, queryErr) + + wantLen := len(names) + if testCase.mixed { + wantLen++ + } + + require.Len(t, results, wantLen) + assert.True(t, fixture.filtered && processed && queried) + assert.Equal(t, 1, fixture.fullClones, "clone project objects only once for the whole query") + assert.Equal(t, len(names), fixture.sharedClones) + assert.Equal(t, len(names), fixture.repacked) + + releases := map[string]string{testComponentName: "3", "rust-asyncband": "3", "bash": "1.azl4"} + for _, result := range results { + assert.Equal(t, releases[result.Name], result.Version.Release()) + } + }) + } +} + func TestComponentQueryCmd_NoMatch(t *testing.T) { const testComponentName = "test-component" diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit.go b/internal/app/azldev/cmds/component/refreshupstreamcommit.go index 02da3bddf..6e99dc46d 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit.go @@ -54,6 +54,10 @@ files before component-specific TOML configuration so subsequent commands use the generated commit unless the component configuration explicitly overrides it. +Only components listed in lockless-components.txt +beside the top-level azldev.toml are refreshed. Other components are skipped +and their generated TOML files are preserved. + All TOML configuration is resolved before the source type is checked. Selected components whose effective spec.type is not "upstream" do not contact an upstream provider, and any generated upstream-commit TOML for them is removed. @@ -75,19 +79,19 @@ The --check-only flag runs the full pipeline but does NOT write TOML files or prune orphans. The command exits 0 when nothing would change and exits 1 when any component is stale or any generated TOML would be pruned. Intended for CI gates.`, Example: ` # Refresh all components - azldev --without-lockfile component refresh-upstream-commit -a + azldev component refresh-upstream-commit -a # Refresh a single component - azldev --without-lockfile component refresh-upstream-commit -p curl + azldev component refresh-upstream-commit -p curl # Refresh components in a group - azldev --without-lockfile component refresh-upstream-commit -g core + azldev component refresh-upstream-commit -g core # Write generated files to a custom directory - azldev --without-lockfile component refresh-upstream-commit -a --upstream-commits-dir config/commits + azldev component refresh-upstream-commit -a --upstream-commits-dir config/commits # CI gate: exit 0 if commit TOMLs are current, 1 if anything would change - azldev --without-lockfile component refresh-upstream-commit -a --check-only -q`, + azldev component refresh-upstream-commit -a --check-only -q`, RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { options.ComponentFilter.ComponentNamePatterns = append( args, options.ComponentFilter.ComponentNamePatterns..., @@ -134,6 +138,8 @@ func RefreshUpstreamCommits( env *azldev.Env, options *RefreshUpstreamCommitOptions, ) ([]RefreshUpstreamCommitResult, error) { resolver := components.NewResolver(env) + options.ComponentFilter.SkipLockValidation = true + resolver.SuppressLockWarnings = true resolved, err := resolver.FindComponents(&options.ComponentFilter) if err != nil { @@ -164,7 +170,9 @@ func RefreshUpstreamCommits( // selected component has an upstream identity. Non-upstream components // never contact a provider; an existing generated pin is marked for // removal instead. - upstreamComps, results, inspectErr := inspectSelectedComponents(allComps, store) + upstreamComps, results, inspectErr := inspectSelectedComponents( + componentsForLockMode(env, allComps, true), store, + ) if inspectErr != nil { return results, inspectErr } @@ -201,7 +209,7 @@ func RefreshUpstreamCommits( // spec-glob-discovered components that aren't in config directly. // Generated TOMLs are version controlled, so pruning is safe even if the // resolved set is empty (e.g., all components removed from config). - wouldPrune, orphanErr := handleOrphanConfigs(store, allComps, options) + wouldPrune, orphanErr := handleOrphanConfigs(env, store, allComps, options) if orphanErr != nil { return filterRefreshDisplayResults(results), orphanErr } @@ -278,12 +286,21 @@ func excludePendingRemovals( // touching disk. Returns (nil, nil) when not running with --all-components, // since orphan handling is scoped to whole-set updates. func handleOrphanConfigs( + env *azldev.Env, store *upstreamcommit.Store, comps []components.Component, options *RefreshUpstreamCommitOptions, ) ([]string, error) { + var ignored []string + + for _, comp := range comps { + if !env.ComponentWithoutLockfile(comp.GetName()) { + ignored = append(ignored, comp.GetName()) + } + } + return handleOrphans( - upstreamCommitOrphanStore{store: store}, + upstreamCommitOrphanStore{store: store, ignored: ignored}, comps, options.ComponentFilter.IncludeAllComponents, options.CheckOnly, @@ -296,13 +313,14 @@ func handleOrphanConfigs( // upstreamCommitOrphanStore adapts [upstreamcommit.Store] to [orphanStore]. type upstreamCommitOrphanStore struct { - store *upstreamcommit.Store + store *upstreamcommit.Store + ignored []string } func (s upstreamCommitOrphanStore) findOrphans( resolved map[string]projectconfig.ComponentConfig, ) ([]string, error) { - orphans, err := s.store.FindOrphans(resolved) + orphans, err := s.store.FindOrphans(resolved, s.ignored...) if err != nil { return nil, fmt.Errorf("finding orphan upstream commit TOMLs:\n%w", err) } @@ -313,7 +331,7 @@ func (s upstreamCommitOrphanStore) findOrphans( func (s upstreamCommitOrphanStore) pruneOrphans( resolved map[string]projectconfig.ComponentConfig, ) (int, error) { - pruned, err := s.store.PruneOrphans(resolved) + pruned, err := s.store.PruneOrphans(resolved, s.ignored...) if err != nil { return 0, fmt.Errorf("pruning orphan upstream commit TOMLs:\n%w", err) } @@ -357,7 +375,7 @@ func refreshCheckOnlyResult( } return display, fmt.Errorf("upstream commit TOML files are stale; %s. "+ - "Run 'azldev --without-lockfile component refresh-upstream-commit -a' to refresh", + "Run 'azldev component refresh-upstream-commit -a' to refresh", strings.Join(parts, "; ")) } diff --git a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go index c3e2de99b..5c8be0ad6 100644 --- a/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go +++ b/internal/app/azldev/cmds/component/refreshupstreamcommit_test.go @@ -70,7 +70,7 @@ path = "../../specs/test-component.spec" for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - testEnv := testutils.NewTestEnvWithoutLockfile(t) + testEnv := testutils.NewTestEnv(t) rootConfig := `includes = [ "base/upstream-commits/*.toml", @@ -93,6 +93,9 @@ release-ver = "1.0" fileperms.PublicFile, )) + require.NoError(t, fileutils.WriteFile(testEnv.TestFS, "/project/lockless-components.txt", + []byte("test-component\n"), fileperms.PublicFile)) + if testCase.componentConfig != "" { require.NoError(t, fileutils.WriteFile( testEnv.TestFS, @@ -110,11 +113,7 @@ release-ver = "1.0" testEnv.TestInterfaces.OSEnvFactory, ) - // The command is only registered in lock-file-free mode, which the - // CLI selects by pre-parsing the global flags before registration. - args := []string{ - "--without-lockfile", "component", "refresh-upstream-commit", "--all-components", - } + args := []string{"component", "refresh-upstream-commit", "--all-components"} app.PreParseGlobalFlags(args) componentcmds.OnAppInit(app) @@ -168,6 +167,83 @@ func TestRefreshUpstreamCommits_WritesCommit(t *testing.T) { assert.Equal(t, commit, savedCommit) } +func TestRefreshUpstreamCommits_OnlyRefreshesLocklessComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + setupMockGit(env, "new-commit") + addRefreshUpstreamComponent(env, "curl") + addRefreshUpstreamComponent(env, "rust-asyncband") + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + require.NoError(t, store.Save("curl", "preserved-commit")) + + results, err := componentcmds.RefreshUpstreamCommits( + env.Env, &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "rust-asyncband", results[0].Component) + + commit, exists, err := store.Get("rust-asyncband") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "new-commit", commit) + commit, exists, err = store.Get("curl") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "preserved-commit", commit) +} + +func TestRefreshUpstreamCommits_PreservesLockedPinsDuringOrphanCleanup(t *testing.T) { + for _, checkOnly := range []bool{false, true} { + t.Run(map[bool]string{false: "prune", true: "check-only"}[checkOnly], func(t *testing.T) { + env := testutils.NewTestEnv(t) + addRefreshUpstreamComponent(env, "locked-upstream") + env.Config.Components["locked-local"] = projectconfig.ComponentConfig{ + Name: "locked-local", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: "/project/locked-local.spec", + }, + } + + store := upstreamcommit.NewStore(env.TestFS, testUpstreamCommitsDir) + for _, name := range []string{"locked-local", "locked-upstream"} { + require.NoError(t, store.Save(name, "preserved-commit")) + } + + options := &componentcmds.RefreshUpstreamCommitOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + CheckOnly: checkOnly, + } + results, err := componentcmds.RefreshUpstreamCommits(env.Env, options) + require.NoError(t, err) + assert.Empty(t, results) + + for _, name := range []string{"locked-local", "locked-upstream"} { + commit, exists, loadErr := store.Get(name) + require.NoError(t, loadErr) + assert.True(t, exists) + assert.Equal(t, "preserved-commit", commit) + } + + require.NoError(t, store.Save("removed-component", "orphan-commit")) + + _, err = componentcmds.RefreshUpstreamCommits(env.Env, options) + if checkOnly { + require.ErrorContains(t, err, "removed-component") + assert.NotContains(t, err.Error(), "locked-local") + } else { + require.NoError(t, err) + } + + exists, err := store.Exists("removed-component") + require.NoError(t, err) + assert.Equal(t, checkOnly, exists) + }) + } +} + func TestRefreshUpstreamCommits_ConfigOnlyChangeDoesNotChangeCommitTOML(t *testing.T) { env := testutils.NewTestEnvWithoutLockfile(t) @@ -397,7 +473,7 @@ func TestRefreshUpstreamCommits_CheckOnly_StaleReturnsError(t *testing.T) { require.Error(t, err, "stale TOML must produce a non-nil error in --check-only mode") assert.Contains(t, err.Error(), "stale", "error message should mention staleness") assert.Contains(t, err.Error(), "curl", "error message should name the stale component") - assert.Contains(t, err.Error(), "azldev --without-lockfile component refresh-upstream-commit -a", + assert.Contains(t, err.Error(), "azldev component refresh-upstream-commit -a", "-a-scoped run should suggest the same -a invocation to refresh") // Results slice must be returned alongside the error so structured diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index ff99942bd..ed293bff7 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -18,6 +18,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" @@ -31,22 +32,51 @@ import ( // RenderOptions holds the options for the render command. type RenderOptions struct { componentCommandOptions - ComponentFilter components.ComponentFilter - OutputDir string - OutputDirExplicit bool // True when --output-dir was explicitly passed on the CLI. - FailOnError bool - Force bool - CleanStale bool - CheckOnly bool - RPMDevBumpspec bool + ComponentFilter components.ComponentFilter + OutputDir string + OutputDirExplicit bool // True when --output-dir was explicitly passed on the CLI. + FailOnError bool + Force bool + CleanStale bool + CheckOnly bool + NoChangeRebuild bool + ChangelogComment string + NoGitCommit bool + AllowUncommitted bool + RPMDevBumpspec bool + changelogCommentExplicit bool } -func renderOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(NewRenderCmd(cmdOptionsForApp(app)...)) +func renderOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewRenderCmd()) } +const renderWorktreeHelp = `Before rendering any lockless component, check for uncommitted tracked changes +and untracked files outside 'project.rendered-specs-dir' and in each selected +component's dist-git directory. Other changes under the configured rendered-specs +directory are ignored. Git-ignored files are excluded. +--allow-uncommitted bypasses this check. +--check-only and --no-git-commit do not bypass it. Locked components are unaffected.` + +const renderLocalHelp = `Local lockless components copy local files except the TOML defining the spec path. +Their Release and changelog are never automatically modified, even with --rpmdev-bumpspec. +Unchanged local non-autorelease components must manually update Release and changelog. +Local autorelease components can use --no-change-rebuild to record a timestamp. +Rendered output is automatically committed unless --no-git-commit is provided. +For local components, --changelog-comment supplies only the Git commit message; +without it, rendering collects the message in an editor before replacing output. +--no-git-commit skips this prompt. Output must be inside the project repository.` + +const renderMessageHelp = `Non-autorelease specs use rpmdev-bumpspec -c with --changelog-comment, or edited text. +Autorelease skips bumpspec. Missing messages are collected in an editor before +bumping or replacing output; the final Git commit never prompts. +Autorelease with --no-git-commit needs no message and never prompts. +Message prompts require both stdin and stdout to be interactive TTYs. +Before replacing existing lockless output, compare NEVRs using host rpmspec. +Autorelease requires an equal or higher NEVR; non-autorelease requires a higher NEVR.` + // NewRenderCmd constructs a [cobra.Command] for the "component render" CLI subcommand. -func NewRenderCmd(opts ...CmdOption) *cobra.Command { +func NewRenderCmd() *cobra.Command { var options RenderOptions var cmd *cobra.Command @@ -68,6 +98,30 @@ Unlike prepare-sources, render skips downloading source tarballs from the lookaside cache — only spec files, patches, scripts, and other git-tracked sidecar files are included. Multiple components can be rendered at once. +Render mode is selected per component by lockless-components.txt beside the +top-level azldev.toml. Each nonempty line names one component to render without +its lock file, using configured upstream commits and their TOML history instead. +All other components use lock files. A missing or empty file means all components +use lock files. + +For lockless upstream components, prepare the changelog, then Release, using the +upstream spec and last committed render before writing macros, applying overlays, +and updating sources. Overlays can override the prepared release and changelog. +Compare output; --no-change-rebuild permits unchanged upstream renders and records +a timestamp only for autorelease components (upstream or local). +` + renderMessageHelp + ` +The first commit also includes all pending changes outside the rendered output directory. +Later commits are component-only. --no-git-commit skips staging and committing. +Output must be inside the project repository; configure Git author identity and +install rpmdev-bumpspec. Commit updated upstream-commit TOML before rendering. +--check-only stops before bumping, replacing output, or committing. +Lockless rendering never uses mock or rpmautospec process-distgit. +The spec retains its RPM macros, including %autorelease and %autochangelog. +All prepared files except .git are retained, without filtering or spectool. Host +rpmautospec generate-changelog/calculate-release are used only when needed for +release history or provenance. Provenance is captured before carryover/bumping and inlined with %define. +Lockless output adds no macros file, load directive, or Source9999. Locked components keep their existing behavior.` + "\n\n" + renderLocalHelp + ` + When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to any component in the project config). Per-component dirs that ARE in config @@ -76,7 +130,7 @@ result table accurately reflects which components actually changed on disk. Top-level non-component siblings (e.g. a hand-placed README.md) are preserved. When using a custom output directory (--output-dir), --force is required alongside --clean-stale as a safety measure. This flag is only -valid with -a.`, +valid with -a.` + "\n\n" + renderWorktreeHelp, Example: ` # Render all components (output dir from config) azldev component render -a @@ -84,13 +138,11 @@ valid with -a.`, azldev component render -p curl # Render to a custom directory, allowing removal of existing rendered component directories - azldev component render -a -o rendered/ --force - - # Render all and remove stale directories - azldev component render -a --clean-stale`, + azldev component render -a -o rendered/ --force`, RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { options.ComponentFilter.ComponentNamePatterns = append(args, options.ComponentFilter.ComponentNamePatterns...) options.OutputDirExplicit = cmd.Flags().Changed("output-dir") + options.changelogCommentExplicit = cmd.Flags().Changed("changelog-comment") options.SpecEditor = specEditorFromCommand(cmd) return RenderComponents(env, &options) @@ -101,8 +153,13 @@ valid with -a.`, }, } - addComponentFilterOptions(cmd, &options.ComponentFilter, newCmdOptions(opts...)) + addComponentFilterOptions(cmd, &options.ComponentFilter) + addRenderOptions(cmd, &options) + + return cmd +} +func addRenderOptions(cmd *cobra.Command, options *RenderOptions) { cmd.Flags().StringVarP(&options.OutputDir, "output-dir", "o", "", "output directory for rendered specs (overrides rendered-specs-dir from config)") _ = cmd.MarkFlagDirname("output-dir") @@ -122,6 +179,15 @@ valid with -a.`, "without modifying the output directory. Exits 0 when nothing would change "+ "and 1 when any component would drift. With -a + --clean-stale, also fails "+ "on orphan rendered-spec directories. Intended for CI gates.") + cmd.Flags().BoolVar(&options.NoChangeRebuild, "no-change-rebuild", false, + "allow unchanged upstream renders and local autorelease renders; record a timestamp only for autorelease") + cmd.Flags().StringVar(&options.ChangelogComment, "changelog-comment", "", + "Git commit message for lockless components and verbatim rpmdev-bumpspec -c comment "+ + "for upstream non-autorelease specs (if omitted, prompt in an editor)") + cmd.Flags().BoolVar(&options.NoGitCommit, "no-git-commit", false, + "leave lockless rendered output uncommitted without staging it") + cmd.Flags().BoolVar(&options.AllowUncommitted, "allow-uncommitted", false, + "allow lockless rendering with uncommitted or untracked files in the project repository") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) // --check-only is a read-only diff against on-disk state; --fail-on-error @@ -129,8 +195,6 @@ valid with -a.`, // muddled (CI would fail on stale failures even when on-disk markers // already record them) and forcing a choice keeps the contract crisp. cmd.MarkFlagsMutuallyExclusive("fail-on-error", "check-only") - - return cmd } // RenderResult holds the result of rendering a single component. @@ -140,6 +204,8 @@ type RenderResult struct { Status string `json:"status" table:"Status"` Error string `json:"error,omitempty" table:"Error,omitempty"` Changed bool `json:"changed" table:"Changed"` + renderErr error `json:"-" table:"-"` + lockless bool `json:"-" table:"-"` } // Render status constants. @@ -150,10 +216,11 @@ const ( ) // RenderComponents renders the post-overlay spec and sidecar files for each -// selected component into the output directory. Processing is done in three phases: +// selected component into the output directory. Processing is done in four phases: // 1. Parallel source preparation (clone, overlay, synthetic git) -// 2. Batch mock processing (rpmautospec + spectool in a single chroot call) -// 3. Parallel finishing (filter files, remove .git, copy output) +// 2. Batch mock processing for locked components only +// 3. Parallel finishing (filter locked files, compare without '.git', move output) +// 4. Serial project commits for lockless upstream component output func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, error) { if err := resolveAndValidateOutputDir(env, options); err != nil { return nil, err @@ -170,17 +237,15 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, return nil, fmt.Errorf("failed to resolve components:\n%w", err) } - if comps.Len() == 0 { - return nil, errors.New("no components were selected; " + - "please use command-line options to indicate which components to render", - ) + componentList := comps.Components() + + if err := validateRenderComponents(env, componentList, options.AllowUncommitted); err != nil { + return nil, err } - // Create mock processor for rpmautospec/spectool. - mockProcessor := createMockProcessor(env) - if mockProcessor == nil { - return nil, errors.New( - "mock config required for rendering; ensure the project has a valid distro with mock config") + mockProcessor, err := renderMockProcessor(env, componentList) + if err != nil { + return nil, err } defer destroyMockProcessor(env, mockProcessor) @@ -204,19 +269,18 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, } }() - componentList := comps.Components() results := make([]*RenderResult, len(componentList)) // ── Phase 1: Parallel source preparation ── prepared := parallelPrepare(env, mockProcessor, componentList, stagingDir, options, results) - // ── Phase 2: Batch mock processing ── - mockResultMap := batchMockProcess(env, mockProcessor, stagingDir, prepared) + // ── Phase 2: Mode-specific source processing ── + mockResultMap := processRenderSources(env, mockProcessor, stagingDir, prepared) // Prune orphan component dirs (components removed from config) when // --clean-stale is set. Per-component output dirs that match the resolved // component set are NOT touched here; phase 3 will RemoveAll+rewrite each - // of them via copyRenderedOutput. This keeps the diff against existing + // of them via moveRenderedOutput. This keeps the diff against existing // output meaningful (so result.Changed reflects actual content drift // instead of being unconditionally true after a blanket wipe) and reduces // the blast radius of a Ctrl-C: we only ever delete dirs that wouldn't @@ -237,16 +301,13 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, // ── Phase 3: Parallel finishing ── parallelFinish(env, prepared, mockResultMap, results, stagingDir, - options.Force, options.CheckOnly) - - // Write RENDER_FAILED markers for any component that errored in phase 1 - // (source preparation) or phase 3 (mock result application + copy). - // Centralizing this here makes it idempotent with the --clean-stale wipe - // (which sits between phases 2 and 3) and keeps the per-phase code free - // of bookkeeping. In --check-only mode this verifies that on-disk state - // matches the expected single-marker shape and flags drift on mismatch. + options.Force, options.CheckOnly, options.NoChangeRebuild) + + // Only locked failures write markers; lockless failures preserve output. writeFailureMarkers(env.FS(), results, options.Force, options.CheckOnly) + commitLocklessRenders(env, prepared, results, options.CheckOnly) + // Sort results alphabetically for consistent output. sortRenderResults(results) @@ -329,15 +390,23 @@ func sortRenderResults(results []*RenderResult) { }) } -// checkRenderErrors counts error and cancelled results and returns an error if FailOnError is set. +// checkRenderErrors reports no-change refusals and NEVR validation errors regardless of failOnError, and +// other error or cancelled results only when failOnError is set. func checkRenderErrors(results []*RenderResult, failOnError bool) error { - var errCount, cancelledCount int + var ( + errCount, cancelledCount int + noChangeErrors []error + ) for _, result := range results { if result == nil { continue } + if errors.Is(result.renderErr, errNoRenderChanges) || errors.Is(result.renderErr, errRenderNEVRValidation) { + noChangeErrors = append(noChangeErrors, fmt.Errorf("%#q: %w", result.Component, result.renderErr)) + } + switch result.Status { case renderStatusError: errCount++ @@ -354,6 +423,10 @@ func checkRenderErrors(results []*RenderResult, failOnError bool) error { slog.Error("Some components failed to render", "errorCount", errCount, "cancelledCount", cancelledCount) + if len(noChangeErrors) > 0 { + return errors.Join(noChangeErrors...) + } + if failOnError { return fmt.Errorf("%d component(s) failed to render", failCount) } @@ -368,6 +441,10 @@ type preparedComponent struct { comp components.Component specFilename string // e.g., "curl.spec" compOutputDir string // validated output path computed in phase 1 + update *locklessRenderUpdate + lockless bool + outputTouched bool + outputExisted bool } // prepResult pairs a prepared component (on success) or a render result (on error). @@ -412,7 +489,8 @@ func parallelPrepare( //nolint:contextcheck // env carries the ctx return prepareOneComponent( workerEnv, mockProcessor, comp, stagingDir, options.OutputDir, - options.specEditorMode(), options.RPMDevBumpspec) + options.specEditorMode(), options.RPMDevBumpspec, + env.ComponentWithoutLockfile(comp.GetName())) }, ) @@ -434,11 +512,18 @@ func parallelPrepare( OutputDir: compOutputDir, Status: renderStatusCancelled, Error: "context cancelled", + lockless: env.ComponentWithoutLockfile(compName), } case result.Value.result != nil: results[idx] = result.Value.result default: result.Value.prepared.index = idx + if update := result.Value.prepared.update; update != nil { + update.subject = options.ChangelogComment + update.subjectProvided = options.changelogCommentExplicit || options.ChangelogComment != "" + update.noGitCommit = options.NoGitCommit + } + prepared = append(prepared, result.Value.prepared) } } @@ -461,6 +546,7 @@ func prepareOneComponent( outputDir string, specEditor spec.EditorMode, rpmDevBumpspec bool, + lockless bool, ) prepResult { componentName := comp.GetName() @@ -472,10 +558,12 @@ func prepareOneComponent( OutputDir: "(invalid)", Status: renderStatusError, Error: nameErr.Error(), + lockless: lockless, }} } - prep, err := prepareComponentSources(env, mockProcessor, comp, stagingDir, specEditor, rpmDevBumpspec) + prep, err := prepareComponentSources(env, mockProcessor, comp, stagingDir, compOutputDir, + specEditor, rpmDevBumpspec, lockless) if err != nil { slog.Error("Failed to prepare component sources", "component", componentName, "error", err) @@ -485,6 +573,7 @@ func prepareOneComponent( OutputDir: compOutputDir, Status: renderStatusError, Error: err.Error(), + lockless: lockless, }} } @@ -501,8 +590,10 @@ func prepareComponentSources( mockProcessor *sources.MockProcessor, comp components.Component, stagingDir string, + compOutputDir string, specEditor spec.EditorMode, rpmDevBumpspec bool, + lockless bool, ) (*preparedComponent, error) { componentName := comp.GetName() @@ -529,16 +620,35 @@ func prepareComponentSources( } // Prepare sources with overlays, skipping lookaside downloads. - // WithGitRepo preserves upstream .git and creates synthetic history so - // rpmautospec can expand %autorelease and %autochangelog correctly. + // Preserve upstream .git for release queries. Only locked rendering creates + // synthetic history for rpmautospec expansion. // WithSkipLookaside avoids expensive tarball downloads — only spec + // sidecar files are needed for rendering. - preparerOpts := append(gitRepoPreparerOptions(env, distro), + preparerOpts := append(gitRepoPreparerOptionsForMode(env, distro, lockless), sources.WithSkipLookaside(), sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), - sources.WithMockProcessor(mockProcessor), ) - if rpmDevBumpspec { + if lockless { + preparerOpts = append(preparerOpts, sources.WithoutSyntheticHistory(), sources.WithSkipLocalSpecConfig(), + sources.WithAutoreleaseResolver(hostRenderProcessor{env: env})) + } else { + preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) + } + + var update *locklessRenderUpdate + + if lockless { + preparerOpts = append(preparerOpts, sources.WithFetchedSources( + func(_ context.Context, fetched components.Component, dir string) error { + var prepErr error + //nolint:contextcheck // The callback context is the captured preparation env. + update, prepErr = prepareFetchedLocklessRender(env, fetched, dir, compOutputDir, specEditor) + + return prepErr + })) + } + + if rpmDevBumpspec && (!lockless || comp.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeLocal) { preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) } @@ -563,6 +673,8 @@ func prepareComponentSources( return &preparedComponent{ comp: comp, specFilename: filepath.Base(specPath), + update: update, + lockless: lockless, }, nil } @@ -573,7 +685,7 @@ func prepareComponentSources( // batchMockProcess runs rpmautospec and spectool for all prepared components in // a single mock chroot invocation. Returns a map from component name to result. // -// NOTE: All components share one mock chroot initialized from the project's +// NOTE: All locked components share one mock chroot initialized from the project's // default distro. Phase 1 resolves distro per-component for source fetching, // but the mock environment (macros, rpmautospec version, etc.) is uniform. // This matches the current Koji build model where all components target the @@ -617,10 +729,8 @@ func batchMockProcess( // Phase 3: Parallel finishing // ────────────────────────────────────────────────────────────────────────────── -// parallelFinish applies mock results (file filtering, .git removal) and copies -// rendered output for all successfully prepared components. In --check-only -// mode, the copy step is replaced with a tree comparison and no disk writes -// happen. +// parallelFinish applies source-query results, compares without '.git', then +// bumps and moves rendered output. Check-only stops after comparison. func parallelFinish( env *azldev.Env, prepared []*preparedComponent, @@ -629,6 +739,7 @@ func parallelFinish( stagingDir string, allowOverwrite bool, checkOnly bool, + noChangeRebuild bool, ) { if len(prepared) == 0 { return @@ -644,11 +755,12 @@ func parallelFinish( parmapResults := parmap.Map( workerEnv, - env.IOBoundConcurrency(), + renderFinishConcurrency(env, prepared, checkOnly), prepared, func(done, _ int) { progressEvent.SetProgress(int64(done), total) }, func(_ context.Context, prep *preparedComponent) *RenderResult { - return finishOneComponent(workerEnv, env, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) + //nolint:contextcheck // workerEnv is the captured context supplied to parmap. + return finishOneComponent(workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly, noChangeRebuild) }, ) @@ -663,6 +775,7 @@ func parallelFinish( OutputDir: prep.compOutputDir, Status: renderStatusCancelled, Error: "context cancelled", + lockless: prep.lockless, } default: results[prep.index] = result.Value @@ -675,12 +788,12 @@ func parallelFinish( // worker; semaphore acquisition is handled by parmap. func finishOneComponent( workerEnv *azldev.Env, - env *azldev.Env, prep *preparedComponent, mockResultMap map[string]*sources.ComponentMockResult, stagingDir string, allowOverwrite bool, checkOnly bool, + noChangeRebuild bool, ) *RenderResult { componentName := prep.comp.GetName() compOutputDir := prep.compOutputDir @@ -693,6 +806,7 @@ func finishOneComponent( OutputDir: compOutputDir, Status: renderStatusCancelled, Error: "context cancelled", + lockless: prep.lockless, } } @@ -700,15 +814,18 @@ func finishOneComponent( Component: componentName, OutputDir: compOutputDir, Status: renderStatusOK, + lockless: prep.lockless, } - drifted, err := finishComponentRender(env, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly) + drifted, err := finishComponentRender( + workerEnv, prep, mockResultMap, stagingDir, allowOverwrite, checkOnly, noChangeRebuild) if err != nil { slog.Error("Failed to finish rendering component", "component", componentName, "error", err) result.Status = renderStatusError result.Error = err.Error() + result.renderErr = err } result.Changed = drifted @@ -716,15 +833,14 @@ func finishOneComponent( return result } -// finishComponentRender applies mock results, filters unreferenced files, -// removes .git, diffs the staging tree against the existing on-disk output, -// and (unless checkOnly is set) copies the staging tree to the output dir. +// finishComponentRender applies mock results, filters locked components' files, +// diffs the staging tree against the existing output with mode-specific exclusions, +// and (unless checkOnly is set) bumps lockless upstream specs and publishes output. // stagingDir is the root staging directory containing the component's subdirectory. // -// Returns changed=true when the staging tree differs from the existing output -// (or no existing output is present). The diff is computed unconditionally so -// every render run gets a meaningful 'Changed' value in its result table; the -// disk write is the only thing gated by checkOnly. +// Returns changed=true when the prepared tree differs from the existing output +// (or no existing output is present), or a subsequent release bump changes the spec. +// Check-only compares before bumping and does not replace output. func finishComponentRender( env *azldev.Env, prep *preparedComponent, @@ -732,12 +848,13 @@ func finishComponentRender( stagingDir string, allowOverwrite bool, checkOnly bool, + noChangeRebuild bool, ) (bool, error) { componentName := prep.comp.GetName() componentDir := filepath.Join(stagingDir, componentName) specPath := filepath.Join(componentDir, prep.specFilename) - // Check mock result. + // Check the source-processing result. mockResult, hasMockResult := mockResultMap[componentName] if !hasMockResult { return false, fmt.Errorf( @@ -746,15 +863,22 @@ func finishComponentRender( if mockResult.Error != nil { return false, fmt.Errorf( - "mock processing failed for %#q:\n%w", componentName, mockResult.Error) + "source processing failed for %#q:\n%w", componentName, mockResult.Error) + } + + if !prep.lockless { + removeRenderedGitMetadata(env, componentDir) } - // Filter files using spectool result from batch mock. + // Filter locked components using the mock spectool result. // Skip filtering when: - // 1. The component config explicitly opts out via 'skip-file-filter'. - // 2. spectool output contains unexpanded RPM macros (%{...}), indicating + // 1. The component is lockless. + // 2. The component config explicitly opts out via 'skip-file-filter'. + // 3. spectool output contains unexpanded RPM macros (%{...}), indicating // that the reported filenames don't match the real files on disk. - if prep.comp.GetConfig().Render.SkipFileFilter { + if prep.lockless { + slog.Debug("Preserving all files for lockless render", "component", componentName) + } else if prep.comp.GetConfig().Render.SkipFileFilter { slog.Info("Skipping file filter ('skip-file-filter' is set)", "component", componentName) } else if macro := findUnexpandedMacro(mockResult.SpecFiles); macro != "" { slog.Info("Skipping file filter (spectool output contains unexpanded macros)", @@ -765,17 +889,8 @@ func finishComponentRender( return false, fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr) } - // Remove .git directory — must not appear in rendered output. - // rpmautospec already read it during the batch mock phase. - gitDir := filepath.Join(componentDir, ".git") - if removeErr := env.FS().RemoveAll(gitDir); removeErr != nil { - slog.Debug("Failed to remove .git directory", "path", gitDir, "error", removeErr) - } - - // Compare staging tree to existing output. Always done so the result table - // reflects which components actually changed on disk this run, not just - // in --check-only mode. - changed, diffErr := diffRenderedOutput(env.FS(), componentDir, prep.compOutputDir) + // Compare before the final bump so '--check-only' does not manufacture release drift. + changed, diffErr := diffRenderedOutput(env.FS(), componentDir, prep.compOutputDir, prep.lockless) if diffErr != nil { return false, fmt.Errorf("comparing rendered output for %#q:\n%w", componentName, diffErr) } @@ -784,9 +899,18 @@ func finishComponentRender( return changed, nil } - // Copy rendered files to the component's output directory. - if copyErr := copyRenderedOutput(env, componentDir, prep.compOutputDir, allowOverwrite); copyErr != nil { - return changed, copyErr + if prep.update != nil { + bumped, err := finishLocklessSpec(env, specPath, prep.update, changed, noChangeRebuild) + if err != nil { + return changed, err + } + + changed = changed || bumped + } + + // Move rendered files to the component's output directory. + if moveErr := moveRenderedOutput(env, componentDir, prep, allowOverwrite); moveErr != nil { + return changed, moveErr } // Best-effort: create a sibling symlink at the URL-encoded component name to @@ -798,10 +922,8 @@ func finishComponentRender( // //nolint:godox // tracked by TODO(koji-fragment-decode) tag. // TODO(koji-fragment-decode): remove once the build system decodes fragments. - if aliasErr := writeAliasSymlink(env.FS(), prep.compOutputDir, componentName); aliasErr != nil { - slog.Warn("Failed to create rendered-spec alias symlink; downstream build steps"+ - " that consume the percent-encoded path may fail to locate this component", - "component", componentName, "error", aliasErr) + if !prep.lockless { + writeRenderedAlias(env, prep) } slog.Info("Rendered component", "component", componentName, @@ -810,6 +932,15 @@ func finishComponentRender( return changed, nil } +func writeRenderedAlias(env *azldev.Env, prep *preparedComponent) { + name := prep.comp.GetName() + if err := writeAliasSymlink(env.FS(), prep.compOutputDir, name, prep.lockless); err != nil { + slog.Warn("Failed to create rendered-spec alias symlink; downstream build steps"+ + " that consume the percent-encoded path may fail to locate this component", + "component", name, "error", err) + } +} + // writeAliasSymlink creates a sibling symlink alongside componentOutputDir at the // URL-encoded form of componentName, pointing back at the real directory with a // relative target. @@ -817,12 +948,8 @@ func finishComponentRender( // No-ops when no encoding is needed (plain ASCII names) or when the underlying // filesystem doesn't support symlinks (e.g., in-memory test FS). // -// Refuses to overwrite a non-symlink at the alias path — if a real component -// directory already lives there (the hypothetical 'gtk%2B' next to 'gtk+' -// case), bail with an error rather than silently destroying that component's -// rendered output. RPM names don't use '%' in practice, so this is belt-and -// suspenders. -func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName string) error { +// Only lockless rendering replaces non-symlink entries at the alias path. +func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName string, replaceExisting bool) error { aliasName := components.RenderedSpecDirAliasName(componentName) if aliasName == "" { return nil @@ -839,23 +966,18 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st parentDir := filepath.Dir(componentOutputDir) aliasPath := filepath.Join(parentDir, aliasName) - // Inspect any existing entry at the alias path. We only ever clobber a - // pre-existing symlink (a stale alias from a previous render); a real - // directory or file there means a name collision with another component - // and must be reported, not silently destroyed. - info, lstatErr := lstatIfPossible(fileSystem, aliasPath) - switch { - case lstatErr == nil && info.Mode()&os.ModeSymlink == 0: - return fmt.Errorf( - "alias path %#q is already occupied by a non-symlink entry; refusing to overwrite", - aliasPath) - case lstatErr == nil: - // Existing symlink — remove and replace below. - if removeErr := fileSystem.Remove(aliasPath); removeErr != nil { - return fmt.Errorf("removing existing alias symlink %#q:\n%w", aliasPath, removeErr) + if !replaceExisting { + info, err := lstatIfPossible(fileSystem, aliasPath) + switch { + case err == nil && info.Mode()&os.ModeSymlink == 0: + return fmt.Errorf("alias path %#q is already occupied by a non-symlink entry; refusing to overwrite", aliasPath) + case err != nil && !errors.Is(err, os.ErrNotExist): + return fmt.Errorf("inspecting alias path %#q:\n%w", aliasPath, err) } - case !errors.Is(lstatErr, os.ErrNotExist): - return fmt.Errorf("inspecting alias path %#q:\n%w", aliasPath, lstatErr) + } + + if removeErr := fileSystem.RemoveAll(aliasPath); removeErr != nil { + return fmt.Errorf("removing existing alias path %#q:\n%w", aliasPath, removeErr) } // Use a relative target so the rendered tree stays portable. @@ -867,34 +989,48 @@ func writeAliasSymlink(fileSystem opctx.FS, componentOutputDir, componentName st return nil } -// lstatIfPossible returns the link info at path without following symlinks, if -// the underlying filesystem supports it. Falls back to a regular Stat otherwise. func lstatIfPossible(fileSystem opctx.FS, path string) (os.FileInfo, error) { if lstater, ok := fileSystem.(afero.Lstater); ok { info, _, err := lstater.LstatIfPossible(path) - return info, err //nolint:wrapcheck // pass-through to the caller. + return info, err //nolint:wrapcheck // Preserve the filesystem error. } - return fileSystem.Stat(path) //nolint:wrapcheck // pass-through to the caller. + return fileSystem.Stat(path) //nolint:wrapcheck // Preserve the filesystem error. } -// copyRenderedOutput copies the rendered files from tempDir to the component's output directory. -// For managed output (inside project root), existing output is removed before copying. -// For external output, existing directories cause an error. -func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, allowOverwrite bool) error { +// moveRenderedOutput recreates the output directory and publishes prepared files +// without '.git'. Locked output is copied; lockless output is moved. +func moveRenderedOutput(env *azldev.Env, tempDir string, prep *preparedComponent, allowOverwrite bool) error { + componentOutputDir := prep.compOutputDir + + if env.DryRun() { + slog.Info("Dry run; would replace rendered output", "source", tempDir, "dest", componentOutputDir) + + return nil + } + exists, existsErr := fileutils.DirExists(env.FS(), componentOutputDir) if existsErr != nil { return fmt.Errorf("checking output directory %#q:\n%w", componentOutputDir, existsErr) } - if exists { - if !allowOverwrite { - return fmt.Errorf( - "output directory %#q already exists; use --force to overwrite", - componentOutputDir) + if exists && !allowOverwrite { + return fmt.Errorf( + "output directory %#q already exists; use --force to overwrite", + componentOutputDir) + } + + if exists && prep.lockless { + if err := validateRenderNEVR(env, tempDir, prep); err != nil { + return err } + } + prep.outputExisted = exists + prep.outputTouched = true + + if exists { // Clean up existing rendered output for this component. if removeErr := env.FS().RemoveAll(componentOutputDir); removeErr != nil { return fmt.Errorf("cleaning output directory %#q:\n%w", componentOutputDir, removeErr) @@ -905,20 +1041,55 @@ func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, all return fmt.Errorf("creating output directory %#q:\n%w", componentOutputDir, mkdirErr) } - // Copy all files from temp to output. - copyOptions := fileutils.CopyDirOptions{ - CopyFileOptions: fileutils.CopyFileOptions{ - PreserveFileMode: true, - }, + if !prep.lockless { + return copyLockedRenderedOutput(env, tempDir, componentOutputDir) } - if copyErr := fileutils.CopyDirRecursive(env, env.FS(), tempDir, componentOutputDir, copyOptions); copyErr != nil { - return fmt.Errorf("copying rendered files to %#q:\n%w", componentOutputDir, copyErr) + return moveLocklessRenderedOutput(env.FS(), tempDir, componentOutputDir) +} + +func moveLocklessRenderedOutput(fileSystem afero.Fs, tempDir, componentOutputDir string) error { + entries, err := fileutils.ReadDir(fileSystem, tempDir) + if err != nil { + return fmt.Errorf("reading prepared output directory %#q:\n%w", tempDir, err) + } + + for _, entry := range entries { + if entry.Name() == ".git" { + continue + } + + sourcePath := filepath.Join(tempDir, entry.Name()) + + destPath := filepath.Join(componentOutputDir, entry.Name()) + if moveErr := fileSystem.Rename(sourcePath, destPath); moveErr != nil { + return fmt.Errorf("moving rendered entry %#q to %#q:\n%w", sourcePath, destPath, moveErr) + } + } + + return nil +} + +func copyLockedRenderedOutput(env *azldev.Env, tempDir, outputDir string) error { + removeRenderedGitMetadata(env, tempDir) + + options := fileutils.CopyDirOptions{ + CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}, + } + if err := fileutils.CopyDirRecursive(env, env.FS(), tempDir, outputDir, options); err != nil { + return fmt.Errorf("copying rendered files to %#q:\n%w", outputDir, err) } return nil } +func removeRenderedGitMetadata(env *azldev.Env, tempDir string) { + gitDir := filepath.Join(tempDir, ".git") + if err := env.FS().RemoveAll(gitDir); err != nil { + slog.Debug("Failed to remove .git directory", "path", gitDir, "error", err) + } +} + // findUnexpandedMacro returns the first filename from specFiles that contains // an unexpanded RPM macro (i.e., a literal "%{...}" sequence), or "" if all // macros were resolved. When spectool cannot resolve a macro, it emits the raw @@ -1059,7 +1230,7 @@ func validateOutputDir(outputDir string) error { // pruneOrphanRenderedDirs removes per-component rendered-spec directories // under outputDir that don't correspond to any component in resolvedComps. // Per-component dirs that ARE in resolvedComps are left alone -- phase 3 will -// overwrite them via copyRenderedOutput, and leaving the prior content here +// overwrite them via moveRenderedOutput, and leaving the prior content here // lets the unconditional diff in finishComponentRender produce a meaningful // result.Changed value for the user-visible table. // @@ -1087,14 +1258,14 @@ func pruneOrphanRenderedDirs( return nil } -// writeFailureMarkers walks the final results slice and writes a RENDER_FAILED -// marker into each errored component's output directory. When allowOverwrite -// is set, any pre-existing content at the path is removed first so the marker -// isn't surrounded by stale render output. +// writeFailureMarkers writes a RENDER_FAILED marker for errored locked components. +// Lockless failures preserve output, and fail check-only without requiring markers. +// When allowOverwrite is set, pre-existing locked output is removed first. // // Cancelled components are intentionally skipped — a Ctrl-C is not a render // failure, just an incomplete run, and silently planting markers under those // circumstances would lie in git diff. +// No-change refusals also leave the existing successful output intact. // // In --check-only mode, no marker is written. Instead, the existing on-disk // state for each errored component is verified to be exactly the standard @@ -1106,7 +1277,15 @@ func writeFailureMarkers( fileSystem opctx.FS, results []*RenderResult, allowOverwrite, checkOnly bool, ) { for _, result := range results { - if result == nil || result.Status != renderStatusError { + if result == nil || result.Status != renderStatusError || errors.Is(result.renderErr, errNoRenderChanges) { + continue + } + + if result.lockless { + if checkOnly { + result.Changed = true + } + continue } @@ -1171,8 +1350,9 @@ const renderErrorMarkerContent = "Rendering failed. See azldev logs for details. // the existing on-disk output (actualDir) and returns true when they differ. // A missing actualDir always counts as drift. Symlinks are compared by target // (filesystems without symlink support skip that check; matches production -// render behavior). -func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (bool, error) { +// render behavior). Only lockless rendering ignores the root '.git' entry +// and '.no-change-rebuild' marker on both sides. +func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string, lockless bool) (bool, error) { actualExists, err := fileutils.DirExists(fileSystem, actualDir) if err != nil { return false, fmt.Errorf("checking actual output dir %#q:\n%w", actualDir, err) @@ -1183,7 +1363,12 @@ func diffRenderedOutput(fileSystem opctx.FS, expectedDir, actualDir string) (boo return true, nil } - result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir) + var options []dirdiff.DiffOption + if lockless { + options = append(options, dirdiff.WithIgnoredPaths(".git", noChangeRebuildMarker)) + } + + result, err := dirdiff.DiffDirs(fileSystem, actualDir, expectedDir, options...) if err != nil { return false, fmt.Errorf("diffing %#q vs %#q:\n%w", actualDir, expectedDir, err) } diff --git a/internal/app/azldev/cmds/component/render_history.go b/internal/app/azldev/cmds/component/render_history.go new file mode 100644 index 000000000..5712e1cd2 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_history.go @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "errors" + "fmt" + "os/exec" + "path/filepath" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +// renderNEVRHistory resolves %autorelease specs against the component's +// filtered dist-git history, which is the history lockless builds use. +type renderNEVRHistory interface { + // Committed returns the NEVR of the committed rendered output. + Committed(filename string) (*rpm.SpecInfo, error) + // Rendered returns the NEVR the prepared output would have once committed. + Rendered(preparedDir, filename string) (*rpm.SpecInfo, error) + Remove() error +} + +//nolint:gochecknoglobals // Replaced by isolated NEVR tests that do not run git or rpmautospec. +var openRenderNEVRHistory = func(env *azldev.Env, parent, outputDir string) (renderNEVRHistory, error) { + return newRenderHistoryRepo(env, parent, outputDir) +} + +const renderHistoryProcessedSpec = "azldev-processed.spec" + +type renderHistoryRepo struct { + env *azldev.Env + dir string + tip string +} + +func newRenderHistoryRepo(env *azldev.Env, parent, outputDir string) (repo *renderHistoryRepo, err error) { + if err := prereqs.RequireExecutable(env, "git", nil); err != nil { + return nil, fmt.Errorf("lockless render NEVR validation requires 'git':\n%w", err) + } + + repoRoot, err := git.RunInDir(env, env, env.ProjectDir(), "rev-parse", "--show-toplevel") + if err != nil { + return nil, fmt.Errorf("finding project repository:\n%w", err) + } + + subdir, err := projectSpecRepoRelPath(env, repoRoot, outputDir) + if err != nil { + return nil, err + } + + dir, err := fileutils.MkdirTemp(env.FS(), parent, "azldev-render-history-") + if err != nil { + return nil, fmt.Errorf("creating dist-git history directory:\n%w", err) + } + + repo = &renderHistoryRepo{env: env, dir: dir} + + defer func() { + if err != nil { + fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), dir, &err) + } + }() + + merge, err := git.RunInDir(env, env, repoRoot, "rev-list", "--merges", "-n1", "HEAD") + if err != nil { + return nil, fmt.Errorf("checking project history for merges:\n%w", err) + } + + if merge != "" { + err = repo.filter(repoRoot, filepath.ToSlash(subdir)) + } else { + err = repo.replay(repoRoot, filepath.ToSlash(subdir)) + } + + if err != nil { + return nil, err + } + + return repo, nil +} + +// replay rebuilds the component's 'git filter-repo --subdirectory-filter' +// history without rewriting the whole project. It requires merge-free history, +// where the path-limited commit list matches the commits filter-repo keeps. +// The repository borrows project objects read-only through alternates. +func (r *renderHistoryRepo) replay(repoRoot, subdir string) error { + format, err := git.RunInDir(r.env, r.env, repoRoot, "rev-parse", "--show-object-format") + if err != nil { + return fmt.Errorf("reading project object format:\n%w", err) + } + + objects, err := git.RunInDir(r.env, r.env, repoRoot, "rev-parse", "--path-format=absolute", "--git-path", "objects") + if err != nil { + return fmt.Errorf("finding project objects:\n%w", err) + } + + if _, err := git.RunInDir(r.env, r.env, r.dir, "init", "-q", "--object-format="+format); err != nil { + return fmt.Errorf("creating dist-git history repository:\n%w", err) + } + + if err := fileutils.WriteFile(r.env.FS(), filepath.Join(r.dir, ".git", "objects", "info", "alternates"), + []byte(objects+"\n"), fileperms.PublicFile); err != nil { + return fmt.Errorf("sharing project objects:\n%w", err) + } + + if _, err := r.git("config", "gc.auto", "0"); err != nil { + return fmt.Errorf("disabling history repository maintenance:\n%w", err) + } + + list, err := git.RunInDir(r.env, r.env, repoRoot, "--literal-pathspecs", + "rev-list", "--reverse", "HEAD", "--", subdir) + if err != nil { + return fmt.Errorf("listing component dist-git commits:\n%w", err) + } + + for _, commit := range strings.Fields(list) { + if r.tip, err = r.replayCommit(commit, subdir); err != nil { + return fmt.Errorf("replaying commit %#q:\n%w", commit, err) + } + } + + if r.tip == "" { + return nil + } + + if _, err := r.git("update-ref", "HEAD", r.tip); err != nil { + return fmt.Errorf("updating dist-git history branch:\n%w", err) + } + + return nil +} + +func (r *renderHistoryRepo) replayCommit(commit, subdir string) (string, error) { + raw, err := r.git("cat-file", "commit", commit) + if err != nil { + return "", fmt.Errorf("reading commit:\n%w", err) + } + + tree, err := r.componentTree(commit, subdir) + if err != nil { + return "", err + } + + path := filepath.Join(r.dir, ".git", "azldev-commit") + if err := fileutils.WriteFile(r.env.FS(), path, []byte(rewriteRenderHistoryCommit(raw, tree, r.tip)), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing rewritten commit:\n%w", err) + } + + hash, err := r.git("hash-object", "-t", "commit", "-w", "--", path) + if err != nil { + return "", fmt.Errorf("writing rewritten commit:\n%w", err) + } + + return hash, nil +} + +func (r *renderHistoryRepo) componentTree(commit, subdir string) (string, error) { + entry, err := r.git("ls-tree", "--full-tree", commit, "--", subdir) + if err != nil { + return "", fmt.Errorf("reading component tree:\n%w", err) + } + + if fields := strings.Fields(entry); len(fields) >= 3 && fields[1] == "tree" { + return fields[2], nil + } + + // filter-repo keeps a deletion of the directory as an empty-tree commit. + tree, err := r.git("hash-object", "-t", "tree", "-w", "--stdin") + if err != nil { + return "", fmt.Errorf("writing empty tree:\n%w", err) + } + + return tree, nil +} + +// rewriteRenderHistoryCommit replaces a raw commit's tree and parent and drops +// signatures, as filter-repo does, keeping all other headers and the message. +func rewriteRenderHistoryCommit(raw, tree, parent string) string { + headers, message, _ := strings.Cut(raw, "\n\n") + rewritten := []string{"tree " + tree} + + if parent != "" { + rewritten = append(rewritten, "parent "+parent) + } + + skipping := false + + for _, line := range strings.Split(headers, "\n") { + if !strings.HasPrefix(line, " ") { + name, _, _ := strings.Cut(line, " ") + skipping = slices.Contains([]string{"tree", "parent", "gpgsig", "gpgsig-sha256", "mergetag"}, name) + } + + if !skipping { + rewritten = append(rewritten, line) + } + } + + return strings.Join(rewritten, "\n") + "\n\n" + message + "\n" +} + +// filter uses the same filtering lockless builds use, for histories with merges. +func (r *renderHistoryRepo) filter(repoRoot, subdir string) error { + if err := requireGitFilterRepo(r.env); err != nil { + return err + } + + if _, err := git.RunInDir(r.env, r.env, repoRoot, + "clone", "-q", "--single-branch", "--no-local", "--no-checkout", "--", repoRoot, r.dir); err != nil { + return fmt.Errorf("cloning project repository:\n%w", err) + } + + if _, err := r.git("filter-repo", "--force", "--subdirectory-filter", subdir); err != nil { + return fmt.Errorf("filtering component dist-git history:\n%w", err) + } + + // An empty filtered history leaves no HEAD commit; treat it as no history. + if tip, err := r.git("rev-parse", "--verify", "-q", "HEAD"); err == nil { + r.tip = tip + } + + return nil +} + +func (r *renderHistoryRepo) Committed(filename string) (*rpm.SpecInfo, error) { + if r.tip == "" { + return nil, errors.New("no committed dist-git history") + } + + return r.query(filename) +} + +func (r *renderHistoryRepo) Rendered(preparedDir, filename string) (*rpm.SpecInfo, error) { + // Stage the prepared output (git skips its '.git') as the next render commit. + if _, err := r.git("--work-tree="+preparedDir, "add", "-A"); err != nil { + return nil, fmt.Errorf("staging prepared output:\n%w", err) + } + + tree, err := r.git("write-tree") + if err != nil { + return nil, fmt.Errorf("writing prepared output tree:\n%w", err) + } + + previous := "" + if r.tip != "" { + if previous, err = r.git("rev-parse", r.tip+"^{tree}"); err != nil { + return nil, fmt.Errorf("reading committed tree:\n%w", err) + } + } + + // Unchanged output produces no render commit, so the committed history stays current. + if tree != previous { + args := []string{ + "-c", "user.name=azldev", "-c", "user.email=azldev@localhost", + "commit-tree", "--no-gpg-sign", "-m", "azldev render NEVR validation", + } + if r.tip != "" { + args = append(args, "-p", r.tip) + } + + commit, err := r.git(append(args, tree)...) + if err != nil { + return nil, fmt.Errorf("committing prepared output:\n%w", err) + } + + if _, err := r.git("update-ref", "HEAD", commit); err != nil { + return nil, fmt.Errorf("updating dist-git history branch:\n%w", err) + } + + r.tip = commit + } + + return r.query(filename) +} + +// query checks out the history tip and resolves its spec as the mock rpmautospec plugin does. +func (r *renderHistoryRepo) query(filename string) (*rpm.SpecInfo, error) { + // Rebuild the index so every file is written, not only those differing from staged stat data. + for _, args := range [][]string{ + {"read-tree", "--empty"}, {"clean", "-q", "-f", "-d", "-x"}, {"read-tree", "--reset", "-u", r.tip}, + } { + if _, err := r.git(args...); err != nil { + return nil, fmt.Errorf("checking out dist-git history:\n%w", err) + } + } + + if err := prereqs.RequireExecutable(r.env, "rpmautospec", nil); err != nil { + return nil, fmt.Errorf("lockless render NEVR validation requires 'rpmautospec':\n%w", err) + } + + processed := filepath.Join(r.dir, ".git", renderHistoryProcessedSpec) + + raw := exec.CommandContext(r.env, "rpmautospec", "process-distgit", filename, processed) + raw.Dir = r.dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := r.env.Command(raw) + if err != nil { + return nil, fmt.Errorf("creating rpmautospec command:\n%w", err) + } + + if _, err := command.RunAndGetOutput(r.env); err != nil { + return nil, fmt.Errorf("processing %#q with rpmautospec (stderr: %#q):\n%w", filename, stderr.String(), err) + } + + return queryRenderSpecNEVR(r.env, processed, r.dir) +} + +func (r *renderHistoryRepo) Remove() error { + if err := r.env.FS().RemoveAll(r.dir); err != nil { + return fmt.Errorf("removing dist-git history directory %#q:\n%w", r.dir, err) + } + + return nil +} + +func (r *renderHistoryRepo) git(args ...string) (string, error) { + output, err := git.RunInDir(r.env, r.env, r.dir, args...) + if err != nil { + return "", fmt.Errorf("running git in dist-git history:\n%w", err) + } + + return output, nil +} diff --git a/internal/app/azldev/cmds/component/render_history_internal_test.go b/internal/app/azldev/cmds/component/render_history_internal_test.go new file mode 100644 index 000000000..64551bd12 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_history_internal_test.go @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const renderHistoryLogFormat = "--format=%T %an <%ae> %ad %cn <%ce> %cd %s" + +func runTestGit(t *testing.T, dir string, args ...string) string { + t.Helper() + + cmd := exec.CommandContext(t.Context(), "git", append([]string{"-C", dir}, args...)...) + cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=Author", "GIT_AUTHOR_EMAIL=author@example.com", + "GIT_COMMITTER_NAME=Committer", "GIT_COMMITTER_EMAIL=committer@example.com") + + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + + return strings.TrimSpace(string(output)) +} + +func writeTestGitFile(t *testing.T, repo, path, content string) { + t.Helper() + + path = filepath.Join(repo, path) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func commitTestGit(t *testing.T, repo, subject string) { + t.Helper() + + runTestGit(t, repo, "add", "-A") + runTestGit(t, repo, "commit", "-q", "--allow-empty", "-m", subject, "-m", "body for "+subject) +} + +func TestRenderHistoryRepoMatchesFilteredHistory(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git is not installed") + } + + for _, merge := range []bool{false, true} { + t.Run(map[bool]string{false: "replay", true: "filter fallback for merges"}[merge], func(t *testing.T) { + _, filterErr := exec.LookPath("git-filter-repo") + if merge && filterErr != nil { + t.Skip("git-filter-repo is not installed") + } + + root := t.TempDir() + repo := filepath.Join(root, "project") + runTestGit(t, root, "init", "-q", "-b", "main", repo) + + writeTestGitFile(t, repo, "specs/c/curl/curl.spec", "v1") + writeTestGitFile(t, repo, "specs/c/curl-other/curl.spec", "other") + commitTestGit(t, repo, "add curl") + writeTestGitFile(t, repo, "base/comps/curl.toml", "config") + commitTestGit(t, repo, "configure curl") + + if merge { + runTestGit(t, repo, "checkout", "-q", "-b", "side") + writeTestGitFile(t, repo, "specs/c/curl/side.patch", "side") + commitTestGit(t, repo, "side change") + runTestGit(t, repo, "checkout", "-q", "main") + writeTestGitFile(t, repo, "README", "readme") + commitTestGit(t, repo, "main change") + runTestGit(t, repo, "merge", "-q", "--no-ff", "-m", "merge side", "side") + } + + writeTestGitFile(t, repo, "specs/c/curl/curl.spec", "v2") + commitTestGit(t, repo, "update curl") + runTestGit(t, repo, "rm", "-q", "-r", "specs/c/curl") + commitTestGit(t, repo, "remove curl") + writeTestGitFile(t, repo, "specs/c/curl/curl.spec", "v3") + commitTestGit(t, repo, "restore curl") + writeTestGitFile(t, repo, "specs/c/curl-other/curl.spec", "other 2") + commitTestGit(t, repo, "update other") + + testEnv := testutils.NewTestEnv(t) + dryRun := azldev.NewAppDryRunnable(false) + cmdFactory, err := azldev.DefaultCmdFactory(dryRun, testEnv.EventListener) + require.NoError(t, err) + + env := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: repo, Config: testEnv.Config, DryRunnable: dryRun, EventListener: testEnv.EventListener, + Interfaces: azldev.SystemInterfaces{ + CmdFactory: cmdFactory, FileSystemFactory: azldev.DefaultFileSystemFactory(), + OSEnvFactory: azldev.DefaultOSEnvFactory(), + }, + }) + + staging := filepath.Join(root, "staging") + require.NoError(t, os.Mkdir(staging, 0o755)) + + history, err := newRenderHistoryRepo(env, staging, filepath.Join(repo, "specs/c/curl")) + require.NoError(t, err) + + assert.Equal(t, runTestGit(t, repo, "rev-parse", "HEAD:specs/c/curl"), + runTestGit(t, history.dir, "rev-parse", "HEAD^{tree}")) + + replayed := runTestGit(t, history.dir, "log", renderHistoryLogFormat) + if !merge { + assert.Equal(t, []string{"restore curl", "remove curl", "update curl", "add curl"}, + strings.Split(runTestGit(t, history.dir, "log", "--format=%s"), "\n")) + assert.Equal(t, "body for add curl", + runTestGit(t, history.dir, "log", "-1", "--format=%b", "HEAD~3")) + } + + if filterErr == nil { + filtered := filepath.Join(root, "filtered") + runTestGit(t, root, "clone", "-q", "--single-branch", "--no-local", "--", repo, filtered) + runTestGit(t, filtered, "filter-repo", "--force", "--subdirectory-filter", "specs/c/curl") + assert.Equal(t, runTestGit(t, filtered, "log", renderHistoryLogFormat), replayed) + } + + // The project repository is only read; the replay writes its own objects. + assert.Empty(t, runTestGit(t, repo, "status", "--porcelain")) + runTestGit(t, repo, "fsck", "--no-progress") + + require.NoError(t, history.Remove()) + + entries, err := os.ReadDir(staging) + require.NoError(t, err) + assert.Empty(t, entries) + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_host.go b/internal/app/azldev/cmds/component/render_host.go new file mode 100644 index 000000000..9c22ea34f --- /dev/null +++ b/internal/app/azldev/cmds/component/render_host.go @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +type hostRenderProcessor struct { + env *azldev.Env +} + +func (p hostRenderProcessor) GenerateChangelog(ctx context.Context, dir, filename string) (string, error) { + return p.run(ctx, dir, "rpmautospec", "generate-changelog", filename) +} + +func (p hostRenderProcessor) CalculateBaseRelease(ctx context.Context, dir, filename string) (string, error) { + return p.run(ctx, dir, "rpmautospec", "calculate-release", "--number-only", filename) +} + +func (p hostRenderProcessor) CalculateRelease(ctx context.Context, dir, filename string) (string, error) { + return p.run(ctx, dir, "rpmautospec", "calculate-release", "--complete-release", filename) +} + +func (p hostRenderProcessor) ExpandRelease(ctx context.Context, dir, filename string) (string, error) { + out, err := p.run(ctx, dir, "rpmspec", + "-q", "--srpm", "--queryformat", "release=%{RELEASE}\n", + "--define", "_sourcedir "+dir, "--define", "_specdir "+dir, filename) + if err != nil { + return "", err + } + + for _, line := range strings.Split(out, "\n") { + if release, found := strings.CutPrefix(strings.TrimSpace(line), "release="); found && release != "" { + return release, nil + } + } + + return "", fmt.Errorf("no Release field in committed spec query output %#q", out) +} + +func (p hostRenderProcessor) run(ctx context.Context, dir, binary string, args ...string) (string, error) { + if err := prereqs.RequireExecutable(p.env, binary, nil); err != nil { + return "", fmt.Errorf("lockless rendering requires host program %#q:\n%w", binary, err) + } + + raw := exec.CommandContext(ctx, binary, args...) + raw.Dir = dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + cmd, err := p.env.Command(raw) + if err != nil { + return "", fmt.Errorf("creating host %#q command:\n%w", binary, err) + } + + out, err := cmd.RunAndGetOutput(ctx) + if err != nil { + return "", fmt.Errorf("running host %#q (%s):\n%w", binary, stderr.String(), err) + } + + return out, nil +} + +func processRenderSources( + env *azldev.Env, processor *sources.MockProcessor, stagingDir string, prepared []*preparedComponent, +) map[string]*sources.ComponentMockResult { + var locked []*preparedComponent + + results := make(map[string]*sources.ComponentMockResult, len(prepared)) + for _, prep := range prepared { + if !prep.lockless { + locked = append(locked, prep) + + continue + } + + name := prep.comp.GetName() + + results[name] = &sources.ComponentMockResult{Name: name} + } + + for name, result := range batchMockProcess(env, processor, stagingDir, locked) { + results[name] = result + } + + return results +} diff --git a/internal/app/azldev/cmds/component/render_host_internal_test.go b/internal/app/azldev/cmds/component/render_host_internal_test.go new file mode 100644 index 000000000..983913887 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_host_internal_test.go @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os/exec" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestHostRenderAutoreleaseCommands(t *testing.T) { + for _, operation := range []string{"changelog", "base", "provenance"} { + t.Run(operation, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("rpmautospec") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, "/working", cmd.Dir) + + switch operation { + case "changelog": + assert.Equal(t, []string{"rpmautospec", "generate-changelog", "curl.spec"}, cmd.Args) + case "base": + assert.Equal(t, []string{"rpmautospec", "calculate-release", "--number-only", "curl.spec"}, cmd.Args) + case "provenance": + assert.Equal(t, []string{"rpmautospec", "calculate-release", "--complete-release", "curl.spec"}, cmd.Args) + } + + return "unchanged output\n", nil + } + processor := hostRenderProcessor{env: env.Env} + + var ( + out string + err error + ) + + switch operation { + case "changelog": + out, err = processor.GenerateChangelog(env.Env, "/working", "curl.spec") + case "base": + out, err = processor.CalculateBaseRelease(env.Env, "/working", "curl.spec") + case "provenance": + out, err = processor.CalculateRelease(env.Env, "/working", "curl.spec") + } + + require.NoError(t, err) + assert.Equal(t, "unchanged output\n", out) + }) + } +} + +func TestHostRenderExpandRelease(t *testing.T) { + const missingTool = "missing tool" + + for _, phase := range []string{"diagnostics", "missing release", "query failure", missingTool} { + t.Run(phase, func(t *testing.T) { + env := testutils.NewTestEnv(t) + if phase != missingTool { + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + } + + queryErr := errors.New("query failed") + called := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + called = true + + assert.Equal(t, "/committed", cmd.Dir) + assert.Equal(t, []string{ + "rpmspec", "-q", "--srpm", "--queryformat", "release=%{RELEASE}\n", + "--define", "_sourcedir /committed", "--define", "_specdir /committed", "curl.spec", + }, cmd.Args) + + switch phase { + case "query failure": + return "", queryErr + case "missing release": + return "error: %changelog not in descending chronological order\nrelease=\n", nil + default: + return "error: %changelog not in descending chronological order\n" + + "release=9.fc43\nwarning: trailing diagnostic\n", nil + } + } + out, err := (hostRenderProcessor{env: env.Env}).ExpandRelease(env.Env, "/committed", "curl.spec") + + assert.Equal(t, phase != missingTool, called) + + switch phase { + case missingTool: + require.ErrorContains(t, err, "requires host program") + case "query failure": + require.ErrorIs(t, err, queryErr) + case "missing release": + require.ErrorContains(t, err, "no Release field") + default: + require.NoError(t, err) + assert.Equal(t, "9.fc43", out) + } + }) + } +} + +func TestMixedRenderRoutesOnlyLockedComponentsThroughMock(t *testing.T) { + const mockFailure = "mock failure" + + for _, phase := range []string{"success", mockFailure} { + t.Run(phase, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("mock") + + ctrl := gomock.NewController(t) + locked := components_testutils.NewMockComponent(ctrl) + locked.EXPECT().GetName().AnyTimes().Return("curl") + + host := components_testutils.NewMockComponent(ctrl) + host.EXPECT().GetName().AnyTimes().Return("rust-asyncband") + + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + processor, err := renderMockProcessor(env.Env, []components.Component{host, locked}) + require.NoError(t, err) + require.NotNil(t, processor) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected host query during lockless rendering: %v", cmd.Args) + + return "", nil + } + batched := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.Equal(t, "mock", cmd.Args[0]) + + if phase == mockFailure { + return errors.New("mock unavailable") + } + + if strings.Contains(strings.Join(cmd.Args, " "), "render_process.py") { + batched = true + inputs, err := fileutils.ReadFile(env.TestFS, "/stage/inputs.json") + require.NoError(t, err) + assert.JSONEq(t, `[{"name":"curl","specFilename":"curl.spec"}]`, string(inputs)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/stage/results.json", + []byte(`[{"name":"curl","specFiles":"Patch0: locked.patch\n","error":null}]`), + fileperms.PublicFile)) + } + + return nil + } + results := processRenderSources(env.Env, processor, + "/stage", []*preparedComponent{ + {comp: locked, specFilename: "curl.spec"}, + {comp: host, specFilename: "rust-asyncband.spec", lockless: true}, + }) + require.Contains(t, results, "rust-asyncband") + assert.Equal(t, phase != mockFailure, batched) + + require.NoError(t, results["rust-asyncband"].Error) + assert.Empty(t, results["rust-asyncband"].SpecFiles) + + if phase == mockFailure { + assert.NotContains(t, results, "curl") + } else { + require.Contains(t, results, "curl") + assert.Equal(t, []string{"locked.patch"}, results["curl"].SpecFiles) + } + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index 396038acc..321d88d10 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -4,15 +4,25 @@ package component import ( + "errors" + "fmt" "os" + "os/exec" "path/filepath" + "strings" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" ) func TestFindSpecFile(t *testing.T) { @@ -57,6 +67,103 @@ func TestFindSpecFile(t *testing.T) { }) } +func TestMoveRenderedOutput(t *testing.T) { + const ( + workingDir = "/working/curl" + outputDir = "/rendered/c/curl" + ) + + for _, testCase := range []struct { + name string + existing bool + allowOverwrite bool + dryRun bool + wantErr bool + }{ + {name: "new output"}, + {name: "replaces output", existing: true, allowOverwrite: true}, + {name: "requires overwrite permission", existing: true, wantErr: true}, + {name: "dry run preserves both trees", existing: true, allowOverwrite: true, dryRun: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + fs := testEnv.TestFS + env := azldev.NewEnv(t.Context(), azldev.EnvOptions{ + ProjectDir: testEnv.Env.ProjectDir(), Config: testEnv.Config, + Interfaces: testEnv.TestInterfaces, EventListener: testEnv.EventListener, + DryRunnable: azldev.NewAppDryRunnable(testCase.dryRun), + }) + + files := map[string]string{ + "curl.spec": "Name: curl\n", ".gitignore": "upstream ignore rules", + "extras/nested.patch": "nested sidecar", + } + for name, content := range files { + path := filepath.Join(workingDir, name) + require.NoError(t, fileutils.MkdirAll(fs, filepath.Dir(path))) + require.NoError(t, fileutils.WriteFile(fs, path, []byte(content), fileperms.PublicFile)) + } + + require.NoError(t, fileutils.MkdirAll(fs, filepath.Join(workingDir, "empty"))) + require.NoError(t, fileutils.MkdirAll(fs, filepath.Join(workingDir, ".git"))) + gitHead := filepath.Join(workingDir, ".git/HEAD") + require.NoError(t, fileutils.WriteFile(fs, gitHead, []byte("upstream history"), fileperms.PublicFile)) + + oldPath := filepath.Join(outputDir, "stale/nested.txt") + if testCase.existing { + require.NoError(t, fileutils.MkdirAll(fs, filepath.Dir(oldPath))) + require.NoError(t, fileutils.WriteFile(fs, oldPath, []byte("old output"), fileperms.PublicFile)) + } + + err := moveRenderedOutput(env, workingDir, &preparedComponent{compOutputDir: outputDir}, testCase.allowOverwrite) + if testCase.wantErr { + require.ErrorContains(t, err, "use --force to overwrite") + } else { + require.NoError(t, err) + } + + readDir := outputDir + if testCase.wantErr || testCase.dryRun { + readDir = workingDir + } + + for name, want := range files { + content, err := fileutils.ReadFile(fs, filepath.Join(readDir, name)) + require.NoError(t, err) + assert.Equal(t, want, string(content)) + } + + emptyExists, err := fileutils.DirExists(fs, filepath.Join(readDir, "empty")) + require.NoError(t, err) + assert.True(t, emptyExists) + + if testCase.wantErr || testCase.dryRun { + head, err := fileutils.ReadFile(fs, gitHead) + require.NoError(t, err) + assert.Equal(t, "upstream history", string(head)) + + old, err := fileutils.ReadFile(fs, oldPath) + require.NoError(t, err) + assert.Equal(t, "old output", string(old)) + + return + } + + for name, want := range files { + content, err := fileutils.ReadFile(fs, filepath.Join(workingDir, name)) + require.NoError(t, err) + assert.Equal(t, want, string(content), "locked publication copies instead of moving") + } + + for _, path := range []string{oldPath, filepath.Join(outputDir, ".git"), gitHead} { + exists, err := fileutils.Exists(fs, path) + require.NoError(t, err) + assert.False(t, exists) + } + }) + } +} + func TestWriteRenderErrorMarker(t *testing.T) { t.Run("creates marker file with static content", func(t *testing.T) { fs := afero.NewMemMapFs() @@ -335,7 +442,7 @@ func TestWriteAliasSymlink(t *testing.T) { realDir := filepath.Join(letterDir, "libxml++") require.NoError(t, fileutils.MkdirAll(osFS, realDir)) - err := writeAliasSymlink(osFS, realDir, "libxml++") + err := writeAliasSymlink(osFS, realDir, "libxml++", false) require.NoError(t, err) aliasPath := filepath.Join(letterDir, "libxml%2B%2B") @@ -354,7 +461,7 @@ func TestWriteAliasSymlink(t *testing.T) { realDir := filepath.Join(letterDir, "vim") require.NoError(t, fileutils.MkdirAll(osFS, realDir)) - err := writeAliasSymlink(osFS, realDir, "vim") + err := writeAliasSymlink(osFS, realDir, "vim", false) require.NoError(t, err) entries, err := fileutils.ReadDir(osFS, letterDir) @@ -373,45 +480,109 @@ func TestWriteAliasSymlink(t *testing.T) { aliasPath := filepath.Join(letterDir, "gtk%2B") osSymlink(t, osFS, "nonexistent-target", aliasPath) - err := writeAliasSymlink(osFS, realDir, "gtk+") + err := writeAliasSymlink(osFS, realDir, "gtk+", false) require.NoError(t, err) assert.Equal(t, "gtk+", osReadlink(t, osFS, aliasPath), "stale symlink should be replaced") }) - t.Run("refuses to overwrite a non-symlink at alias path", func(t *testing.T) { - osFS := afero.NewOsFs() - root := t.TempDir() - letterDir := filepath.Join(root, "g") - realDir := filepath.Join(letterDir, "gtk+") - require.NoError(t, fileutils.MkdirAll(osFS, realDir)) + const symlinkType = "symlink" - // Hypothetical collision: a real component named 'gtk%2B' already exists - // at the alias path. RPM names don't use '%' in practice, but the guard - // must protect against it. - collisionDir := filepath.Join(letterDir, "gtk%2B") - require.NoError(t, fileutils.MkdirAll(osFS, collisionDir)) + for _, entryType := range []string{"file", "directory", symlinkType} { + t.Run("replaces existing "+entryType+" at alias path", func(t *testing.T) { + osFS := afero.NewOsFs() + letterDir := filepath.Join(t.TempDir(), "g") + realDir := filepath.Join(letterDir, "gtk+") + require.NoError(t, fileutils.MkdirAll(osFS, realDir)) - err := writeAliasSymlink(osFS, realDir, "gtk+") - require.Error(t, err) - assert.Contains(t, err.Error(), "non-symlink") + aliasPath := filepath.Join(letterDir, "gtk%2B") + stalePath := aliasPath - // Collision dir must still exist -- the guard MUST NOT have destroyed it. - exists, existsErr := fileutils.DirExists(osFS, collisionDir) - require.NoError(t, existsErr) - assert.True(t, exists, "non-symlink at alias path must be preserved") - }) + switch entryType { + case "directory": + stalePath = filepath.Join(aliasPath, "nested", "stale.txt") + case symlinkType: + stalePath = filepath.Join(letterDir, "old-target", "keep.txt") + } + + require.NoError(t, fileutils.MkdirAll(osFS, filepath.Dir(stalePath))) + require.NoError(t, fileutils.WriteFile(osFS, stalePath, []byte("old contents"), fileperms.PublicFile)) + + if entryType == symlinkType { + osSymlink(t, osFS, "old-target", aliasPath) + } + + require.NoError(t, writeAliasSymlink(osFS, realDir, "gtk+", true)) + linkInfo := osLstat(t, osFS, aliasPath) + assert.NotZero(t, linkInfo.Mode()&os.ModeSymlink) + assert.Equal(t, "gtk+", osReadlink(t, osFS, aliasPath)) + + if entryType == symlinkType { + contents, err := fileutils.ReadFile(osFS, stalePath) + require.NoError(t, err) + assert.Equal(t, "old contents", string(contents), "removing an alias must not remove its target") + } + }) + } t.Run("no-op on filesystem without symlink support", func(t *testing.T) { // MemMapFs does NOT implement afero.Linker -- function should silently no-op. memFS := afero.NewMemMapFs() require.NoError(t, fileutils.MkdirAll(memFS, "/SPECS/l/libxml++")) - err := writeAliasSymlink(memFS, "/SPECS/l/libxml++", "libxml++") + err := writeAliasSymlink(memFS, "/SPECS/l/libxml++", "libxml++", false) assert.NoError(t, err, "must not fail when FS doesn't support symlinks") }) } +type renderAliasTestFS struct { + afero.Fs + linked bool +} + +func (fs *renderAliasTestFS) SymlinkIfPossible(_, _ string) error { + fs.linked = true + + return nil +} + +func TestWriteRenderedAlias_LockModes(t *testing.T) { + for _, lockless := range []bool{false, true} { + for _, directory := range []bool{false, true} { + t.Run(fmt.Sprintf("lockless=%t/directory=%t", lockless, directory), func(t *testing.T) { + env := testutils.NewTestEnv(t) + aliasFS := &renderAliasTestFS{Fs: env.TestFS} + env.TestFS = aliasFS + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("libxml++") + + path := "/output/l/libxml%2B%2B" + if directory { + path = filepath.Join(path, "keep.txt") + } + + require.NoError(t, fileutils.WriteFile(aliasFS, path, []byte("keep"), fileperms.PublicFile)) + + writeRenderedAlias(env.Env, &preparedComponent{ + comp: comp, compOutputDir: "/output/l/libxml++", lockless: lockless, + }) + + assert.Equal(t, lockless, aliasFS.linked) + + exists, err := fileutils.Exists(aliasFS, path) + require.NoError(t, err) + assert.Equal(t, !lockless, exists) + + if !lockless { + content, err := fileutils.ReadFile(aliasFS, path) + require.NoError(t, err) + assert.Equal(t, "keep", string(content)) + } + }) + } + } +} + func TestPruneOrphanRenderedDirs(t *testing.T) { t.Parallel() @@ -581,6 +752,275 @@ func TestWriteFailureMarkers(t *testing.T) { }) } +func TestFinishComponentRender_PreservesGitMetadata(t *testing.T) { + for _, testCase := range []struct { + name string + checkOnly bool + }{ + {name: "normal render"}, + {name: "check-only", checkOnly: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + + for _, root := range []string{"/staging/curl", "/output/curl"} { + for name, content := range map[string]string{ + "curl.spec": "Name: curl\nVersion: 1\nRelease: 1\n", ".gitignore": "unchanged", + } { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, name), + []byte(content), fileperms.PublicFile)) + } + } + + const gitHead = "/staging/curl/.git/HEAD" + + require.NoError(t, fileutils.WriteFile(env.TestFS, gitHead, []byte("upstream"), fileperms.PublicFile)) + + prep := &preparedComponent{ + comp: comp, specFilename: "curl.spec", compOutputDir: "/output/curl", lockless: true, + } + mockResults := map[string]*sources.ComponentMockResult{"curl": {}} + + stubIncreasingRenderNEVR(t, env) + changed, err := finishComponentRender(env.Env, prep, mockResults, "/staging", true, testCase.checkOnly, false) + require.NoError(t, err) + assert.False(t, changed, "Git metadata alone must not count as rendered drift") + + head, err := fileutils.ReadFile(env.TestFS, gitHead) + require.NoError(t, err) + assert.Equal(t, "upstream", string(head)) + + exists, err := fileutils.Exists(env.TestFS, "/output/curl/.git") + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestFinishComponentRender_CheckOnlyMetadata(t *testing.T) { + for _, lockless := range []bool{false, true} { + for _, metadata := range []string{"", ".git/HEAD", noChangeRebuildMarker, "nested/" + noChangeRebuildMarker} { + t.Run(fmt.Sprintf("lockless=%t/%s", lockless, metadata), func(t *testing.T) { + env := testutils.NewTestEnv(t) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Render: projectconfig.ComponentRenderConfig{SkipFileFilter: true}, + }) + + for _, root := range []string{"/staging/curl", "/output/curl"} { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, "curl.spec"), + []byte("Name: curl\n"), fileperms.PublicFile)) + } + + const stagedGit = "/staging/curl/.git/HEAD" + require.NoError(t, fileutils.WriteFile(env.TestFS, stagedGit, []byte("upstream"), fileperms.PublicFile)) + + if metadata != "" { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join("/output/curl", metadata), + []byte("existing metadata"), fileperms.PublicFile)) + } + + prep := &preparedComponent{ + comp: comp, specFilename: "curl.spec", compOutputDir: "/output/curl", lockless: lockless, + } + changed, err := finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, true, false) + require.NoError(t, err) + assert.Equal(t, metadata != "" && (!lockless || strings.HasPrefix(metadata, "nested/")), changed) + + exists, err := fileutils.Exists(env.TestFS, stagedGit) + require.NoError(t, err) + assert.Equal(t, lockless, exists) + + if metadata != "" { + content, err := fileutils.ReadFile(env.TestFS, filepath.Join("/output/curl", metadata)) + require.NoError(t, err) + assert.Equal(t, "existing metadata", string(content), "check-only must not change existing output") + } + }) + } + } +} + +func TestFinishComponentRender_ComparisonBeforeBump(t *testing.T) { + for _, testCase := range []struct { + name string + checkOnly bool + drift bool + autorelease bool + compareFail bool + noChangeRebuild bool + wantNoChanges bool + wantChanged bool + }{ + {name: "check-only unchanged with flag", checkOnly: true, noChangeRebuild: true}, + {name: "check-only drift", checkOnly: true, drift: true, wantChanged: true}, + {name: "unchanged render refused", wantNoChanges: true}, + {name: "forced static bump", noChangeRebuild: true, wantChanged: true}, + {name: "forced autorelease no-op", autorelease: true, noChangeRebuild: true, wantChanged: true}, + {name: "normal drift and no-op", autorelease: true, drift: true, wantChanged: true}, + {name: "drift with flag", drift: true, noChangeRebuild: true, wantChanged: true}, + {name: "comparison fails", compareFail: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + baseFS := env.TestFS + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + + release := "9" + if testCase.autorelease { + release = "%autorelease" + } + + original := renderTestSpec("1", release, "* Previous entry\n") + + actual := original + if testCase.drift { + actual += "\n# Stale output\n" + } + + const ( + stagedSpec = "/staging/curl/curl.spec" + outputSpec = "/output/curl/curl.spec" + timestamp = "2026-11-04T22:53:27+00:00\n" + ) + + require.NoError(t, fileutils.WriteFile(baseFS, stagedSpec, []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(baseFS, outputSpec, []byte(actual), fileperms.PublicFile)) + markerPath := filepath.Join(filepath.Dir(outputSpec), noChangeRebuildMarker) + require.NoError(t, fileutils.WriteFile(baseFS, markerPath, []byte("old timestamp\n"), fileperms.PublicFile)) + + comparisonFS := &renderComparisonFS{Fs: baseFS, path: outputSpec} + if testCase.compareFail { + comparisonFS.readErr = errors.New("comparison read failed") + } + + env.TestFS = comparisonFS + called := false + dated := false + wantFinish := !testCase.checkOnly && !testCase.compareFail && !testCase.wantNoChanges + wantBump := wantFinish && !testCase.autorelease + wantMarker := wantFinish && !testCase.drift && testCase.autorelease + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.True(t, comparisonFS.compared, "comparison must precede the bump") + + called = true + + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + + markerExists, err := fileutils.Exists(baseFS, filepath.Join(filepath.Dir(stagedSpec), noChangeRebuildMarker)) + require.NoError(t, err) + assert.False(t, markerExists, "static release bumps must not create a timestamp") + + bumped := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) + + return fileutils.WriteFile(baseFS, stagedSpec, []byte(bumped), fileperms.PublicFile) + } + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.True(t, comparisonFS.compared) + assert.False(t, called) + assert.Equal(t, []string{"date", "-Is"}, cmd.Args) + assert.Equal(t, filepath.Dir(stagedSpec), cmd.Dir) + + dated = true + + return timestamp, nil + } + if !testCase.checkOnly { + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + if testCase.autorelease { + env.CmdFactory.RegisterCommandInSearchPath("date") + } + } + + prep := &preparedComponent{ + comp: comp, specFilename: "curl.spec", compOutputDir: "/output/curl", + lockless: true, update: &locklessRenderUpdate{subject: "updated curl"}, + } + + stubIncreasingRenderNEVR(t, env) + result := finishOneComponent(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, testCase.checkOnly, testCase.noChangeRebuild) + + err := result.renderErr + + switch { + case testCase.compareFail: + require.ErrorIs(t, err, comparisonFS.readErr) + case testCase.wantNoChanges: + require.ErrorIs(t, err, errNoRenderChanges) + writeFailureMarkers(baseFS, []*RenderResult{result}, true, false) + + for _, failOnError := range []bool{false, true} { + require.ErrorIs(t, checkRenderErrors([]*RenderResult{result}, failOnError), errNoRenderChanges) + } + default: + require.NoError(t, err) + } + + assert.True(t, comparisonFS.compared) + assert.Equal(t, wantBump, called) + assert.Equal(t, wantMarker, dated) + assert.Equal(t, testCase.wantChanged, result.Changed) + + want := strings.Replace(original, "Release: 9\n", "Release: 10\n", 1) + if !wantFinish { + want = actual + staged, readErr := fileutils.ReadFile(baseFS, stagedSpec) + require.NoError(t, readErr) + assert.Equal(t, original, string(staged)) + } else if !wantBump { + want = original + } + + output, readErr := fileutils.ReadFile(baseFS, outputSpec) + require.NoError(t, readErr) + assert.Equal(t, want, string(output)) + + markerExists, readErr := fileutils.Exists(baseFS, markerPath) + require.NoError(t, readErr) + assert.Equal(t, !wantFinish || wantMarker, markerExists) + + if markerExists { + marker, readErr := fileutils.ReadFile(baseFS, markerPath) + require.NoError(t, readErr) + + wantTimestamp := "old timestamp\n" + if wantMarker { + wantTimestamp = timestamp + } + + assert.Equal(t, wantTimestamp, string(marker)) + } + }) + } +} + +type renderComparisonFS struct { + afero.Fs + path string + compared bool + readErr error +} + +func (fs *renderComparisonFS) Open(name string) (afero.File, error) { + if name == fs.path { + fs.compared = true + if fs.readErr != nil { + return nil, fs.readErr + } + } + + return fs.Fs.Open(name) +} + func TestDiffRenderedOutput(t *testing.T) { t.Parallel() @@ -593,7 +1033,7 @@ func TestDiffRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(testFS, "/act/curl.spec", []byte("Name: curl"), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(testFS, "/act/sources", []byte("hash"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/act") + drifted, err := diffRenderedOutput(testFS, "/exp", "/act", false) require.NoError(t, err) assert.False(t, drifted) }) @@ -604,7 +1044,7 @@ func TestDiffRenderedOutput(t *testing.T) { testFS := afero.NewMemMapFs() require.NoError(t, fileutils.WriteFile(testFS, "/exp/curl.spec", []byte("x"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/missing") + drifted, err := diffRenderedOutput(testFS, "/exp", "/missing", false) require.NoError(t, err) assert.True(t, drifted) }) @@ -616,7 +1056,7 @@ func TestDiffRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(testFS, "/exp/curl.spec", []byte("v2"), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(testFS, "/act/curl.spec", []byte("v1"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/act") + drifted, err := diffRenderedOutput(testFS, "/exp", "/act", false) require.NoError(t, err) assert.True(t, drifted) }) @@ -629,12 +1069,51 @@ func TestDiffRenderedOutput(t *testing.T) { require.NoError(t, fileutils.WriteFile(testFS, "/act/curl.spec", []byte("x"), fileperms.PublicFile)) require.NoError(t, fileutils.WriteFile(testFS, "/act/stale.patch", []byte("y"), fileperms.PublicFile)) - drifted, err := diffRenderedOutput(testFS, "/exp", "/act") + drifted, err := diffRenderedOutput(testFS, "/exp", "/act", false) require.NoError(t, err) assert.True(t, drifted) }) } +func TestDiffRenderedOutput_RebuildMarker(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name, existing, working string + wantChanged bool + }{ + {name: "existing marker ignored", existing: noChangeRebuildMarker}, + {name: "working marker ignored", working: noChangeRebuildMarker}, + {name: "different markers ignored", existing: noChangeRebuildMarker, working: noChangeRebuildMarker}, + {name: "nested existing marker compared", existing: "nested/" + noChangeRebuildMarker, wantChanged: true}, + {name: "nested working marker compared", working: "nested/" + noChangeRebuildMarker, wantChanged: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + testFS := afero.NewMemMapFs() + for _, root := range []string{"/existing", "/working"} { + require.NoError(t, fileutils.WriteFile(testFS, filepath.Join(root, "curl.spec"), + []byte("identical"), fileperms.PublicFile)) + } + + if testCase.existing != "" { + require.NoError(t, fileutils.WriteFile(testFS, filepath.Join("/existing", testCase.existing), + []byte("timestamp\n"), fileperms.PublicFile)) + } + + if testCase.working != "" { + require.NoError(t, fileutils.WriteFile(testFS, filepath.Join("/working", testCase.working), + []byte("working timestamp\n"), fileperms.PublicFile)) + } + + changed, err := diffRenderedOutput(testFS, "/working", "/existing", true) + require.NoError(t, err) + assert.Equal(t, testCase.wantChanged, changed) + }) + } +} + func TestOutputDriftsFromMarker(t *testing.T) { t.Parallel() diff --git a/internal/app/azldev/cmds/component/render_lockless.go b/internal/app/azldev/cmds/component/render_lockless.go new file mode 100644 index 000000000..4758b371d --- /dev/null +++ b/internal/app/azldev/cmds/component/render_lockless.go @@ -0,0 +1,1496 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +const noChangeRebuildMarker = ".no-change-rebuild" + +var errNoRenderChanges = errors.New("no change since the last render; use '--no-change-rebuild' to rebuild anyway") + +var errLocalManualRebuild = errors.New( + "local non-autorelease components must manually update their release and changelog before rendering again", +) + +var errRenderNEVRValidation = errors.New("lockless render NEVR validation failed") + +type locklessRenderUpdate struct { + repoRoot string + path string + subject string + changes string + noGitCommit bool + subjectProvided bool + local bool +} + +type lastRender struct { + spec string + tree *object.Tree + previous string + current string + previousBranch string + currentBranch string +} + +type renderAutorelease interface { + GenerateChangelog(ctx context.Context, dir, filename string) (string, error) + CalculateBaseRelease(ctx context.Context, dir, filename string) (string, error) + CalculateRelease(ctx context.Context, dir, filename string) (string, error) + ExpandRelease(ctx context.Context, dir, filename string) (string, error) +} + +//nolint:gochecknoglobals // Replaced by isolated render tests to open an in-memory project repository. +var openRenderRepo = openChangedRepo + +func validateRenderComponents( + env *azldev.Env, comps []components.Component, allowUncommitted bool, +) error { + if len(comps) == 0 { + return errors.New("no components were selected; " + + "please use command-line options to indicate which components to render") + } + + if allowUncommitted { + return nil + } + + for _, comp := range comps { + if env.ComponentWithoutLockfile(comp.GetName()) { + return checkRenderWorktree(env, comps) + } + } + + return nil +} + +func checkRenderWorktree(env *azldev.Env, comps []components.Component) error { + specDir := env.Config().Project.RenderedSpecsDir + if specDir == "" { + specDir = projectconfig.DefaultRenderedSpecsDir + } + + if filepath.IsAbs(specDir) { + relative, err := filepath.Rel(env.ProjectDir(), specDir) + if err != nil { + return fmt.Errorf("resolving rendered-specs directory for repository check:\n%w", err) + } + + specDir = relative + } + + specDir = filepath.Clean(specDir) + componentPaths := make([]string, 0, len(comps)) + + for _, comp := range comps { + name := comp.GetName() + path := filepath.Join(specDir, strings.ToLower(name[:1]), name) + componentPaths = append(componentPaths, ":(literal)"+filepath.ToSlash(path)) + } + + scopes := [][]string{ + {":/", ":(exclude,literal)" + filepath.ToSlash(specDir)}, + componentPaths, + } + for _, scope := range scopes { + args := append([]string{ + "status", "--porcelain=v1", "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...) + + status, err := git.RunInDir(env, env, env.ProjectDir(), args...) + if err != nil { + return fmt.Errorf("checking project repository for uncommitted files:\n%w", err) + } + + if status != "" { + return errors.New("rendering will not be done due to uncommitted or untracked files " + + "outside 'project.rendered-specs-dir' or in selected component dist-git directories; " + + "use '--allow-uncommitted' to proceed") + } + } + + return nil +} + +func prepareLocklessRender( + env *azldev.Env, comp components.Component, + dir, outputDir string, editor spec.EditorMode, +) (*locklessRenderUpdate, error) { + repo, update, err := prepareLocklessRenderOutput(env, outputDir) + if err != nil { + return nil, err + } + + history, err := readLastRender(env, repo, update.repoRoot, update.path, comp.GetName(), false) + if err != nil { + return nil, fmt.Errorf("reading committed render state:\n%w", err) + } + + _, opened, err := openLocklessSpec(env, dir, comp.GetName(), editor) + if err != nil { + return nil, err + } + + version, err := opened.GetLastTag("", "Version") + if err != nil { + return nil, fmt.Errorf("reading upstream Version:\n%w", err) + } + + if err := collectRenderUpstreamChanges(env, dir, comp.GetName(), version, history, update); err != nil { + return nil, err + } + + if err := prepareLocklessSpec(env, hostRenderProcessor{env: env}, dir, comp.GetName(), editor, history); err != nil { + return nil, fmt.Errorf("preparing changelog and release for %#q:\n%w", comp.GetName(), err) + } + + return update, nil +} + +func prepareFetchedLocklessRender( + env *azldev.Env, comp components.Component, dir, outputDir string, editor spec.EditorMode, +) (*locklessRenderUpdate, error) { + if comp.GetConfig().Spec.SourceType != projectconfig.SpecSourceTypeLocal { + return prepareLocklessRender(env, comp, dir, outputDir, editor) + } + + _, update, err := prepareLocklessRenderOutput(env, outputDir) + if update != nil { + update.local = true + } + + return update, err +} + +func prepareLocklessRenderOutput( + env *azldev.Env, outputDir string, +) (repo *gogit.Repository, update *locklessRenderUpdate, err error) { + repo, root, err := openRenderRepo(env) + if err != nil { + return nil, nil, err + } + + path, err := repoRelPath(root, outputDir) + if err != nil { + return nil, nil, fmt.Errorf("lockless render output must be inside the project repository:\n%w", err) + } + + return repo, &locklessRenderUpdate{repoRoot: root, path: path}, nil +} + +func readLastRender( + env *azldev.Env, repo *gogit.Repository, root, path, name string, useWorkingInputs bool, +) (lastRender, error) { + var history lastRender + + head, err := repo.Head() + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return history, nil + } + + if err != nil { + return history, fmt.Errorf("reading project HEAD:\n%w", err) + } + + tree, err := resolveTree(repo, head.Hash().String()) + if err != nil { + return history, err + } + + history.tree, err = tree.Tree(path) + if errors.Is(err, object.ErrDirectoryNotFound) { + return history, nil + } + + if err != nil { + return history, fmt.Errorf("reading committed dist-git directory:\n%w", err) + } + + file, err := tree.File(filepath.ToSlash(filepath.Join(path, name+".spec"))) + if err != nil { + return history, fmt.Errorf("reading committed spec:\n%w", err) + } + + history.spec, err = file.Contents() + if err != nil { + return history, fmt.Errorf("reading committed spec contents:\n%w", err) + } + + lastHash, err := git.RunInDir(env, env, root, + "--literal-pathspecs", "log", "-1", "--format=%H", "--end-of-options", head.Hash().String(), "--", path+"/") + if err != nil { + return history, fmt.Errorf("finding last render commit:\n%w", err) + } + + if lastHash == "" { + return history, nil + } + + last, err := repo.CommitObject(plumbing.NewHash(lastHash)) + if err != nil { + return history, fmt.Errorf("reading last render commit:\n%w", err) + } + + err = readRenderedUpstreamCommits(env, root, name, tree, last, &history, useWorkingInputs) + + return history, err +} + +func readRenderedUpstreamCommits( + env *azldev.Env, root, name string, tree *object.Tree, last *object.Commit, history *lastRender, + useWorkingInputs bool, +) error { + projectPath, err := repoRelPath(root, env.ProjectDir()) + if err != nil { + return err + } + + if useWorkingInputs { + history.current, history.currentBranch, err = configuredRenderUpstreamCommit(env, name, false) + } else { + history.current, history.currentBranch, err = renderUpstreamCommit(env, tree, projectPath, name, false) + } + + if err != nil { + return fmt.Errorf("reading current render upstream commit:\n%w", err) + } + + lastTree, err := last.Tree() + if err == nil { + history.previous, history.previousBranch, err = renderUpstreamCommit(env, lastTree, projectPath, name, true) + } + + if err != nil { + slog.Warn("Cannot read last render upstream commit; treating upstream as changed", + "component", name, "error", err) + + history.previous = "" + history.previousBranch = "" + } + + return nil +} + +func renderUpstreamCommit( + env *azldev.Env, tree *object.Tree, projectPath, name string, allowLockFallback bool, +) (commit string, branch string, err error) { + historicalEnv, err := loadHistoricalEnv(env, tree, projectPath) + if err != nil { + return "", "", err + } + + return configuredRenderUpstreamCommit(historicalEnv, name, allowLockFallback) +} + +func configuredRenderUpstreamCommit( + env *azldev.Env, name string, allowLockFallback bool, +) (commit string, branch string, err error) { + comp, err := components.NewResolver(env).GetComponentByName(name) + if err != nil { + return "", "", fmt.Errorf("resolving component %#q:\n%w", name, err) + } + + commit = comp.GetConfig().Spec.UpstreamCommit + if commit == "" && allowLockFallback { + commit, err = lastRenderLockedUpstreamCommit(env, name) + if err != nil { + return "", "", err + } + } + + if commit == "" { + return "", "", fmt.Errorf("component %#q has no configured 'spec.upstream-commit'", name) + } + + distro, err := sourceproviders.ResolveDistro(env, comp) + if err != nil { + return "", "", fmt.Errorf("resolving upstream branch:\n%w", err) + } + + return commit, distro.Version.DistGitBranch, nil +} + +func lastRenderLockedUpstreamCommit(env *azldev.Env, name string) (string, error) { + selected, err := projectconfig.ReadLocklessComponents(env.FS(), env.ProjectDir()) + if err != nil { + return "", fmt.Errorf("reading historical lockless component selection:\n%w", err) + } + + if selected[name] { + return "", fmt.Errorf("historical component %#q is lockless but has no 'spec.upstream-commit'", name) + } + + lockDir := env.Config().Project.LockDir + if lockDir == "" { + return "", errors.New("no historical 'lock-dir' configured for lock fallback") + } + + lock, err := lockfile.NewStore(env.FS(), lockDir).Get(name) + if err != nil { + return "", fmt.Errorf("reading historical lock for %#q:\n%w", name, err) + } + + if lock.UpstreamCommit == "" { + return "", fmt.Errorf("historical lock for %#q has no 'upstream-commit'", name) + } + + return lock.UpstreamCommit, nil +} + +func openLocklessSpec( + env *azldev.Env, dir, name string, editor spec.EditorMode, +) (path string, opened *spec.Spec, err error) { + path, err = findSpecFile(env.FS(), dir, name) + if err != nil { + return "", nil, err + } + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return "", nil, fmt.Errorf("reading upstream spec:\n%w", err) + } + + opened, err = spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) + if err != nil { + return "", nil, fmt.Errorf("parsing upstream spec:\n%w", err) + } + + return path, opened, nil +} + +func prepareLocklessSpec( + env *azldev.Env, autospec renderAutorelease, dir, name string, editor spec.EditorMode, + history lastRender, +) error { + path, opened, err := openLocklessSpec(env, dir, name, editor) + if err != nil { + return err + } + + original, err := upstreamRenderRelease(opened) + if err != nil { + return err + } + + version, err := opened.GetLastTag("", "Version") + if err != nil { + return fmt.Errorf("reading upstream Version:\n%w", err) + } + + versionChanged, err := locklessVersionChanged(version, history) + if err != nil { + return err + } + + history, err = staticRenderHistory(env, autospec, dir, name, original, history, editor) + if err != nil { + return err + } + + if err := prepareLocklessChangelog(env, autospec, dir, filepath.Base(path), original, history); err != nil { + return err + } + + opened, err = carryLocklessInlineChangelog(opened, history.spec, original, editor) + if err != nil { + return err + } + + if !versionChanged || sources.ReleaseUsesAutorelease(original) { + update, err := locklessRenderRelease(env, autospec, dir, filepath.Base(path), original, history, versionChanged) + if err != nil { + return err + } + + if err := opened.UpdateExistingTagFunc("", "Release", update); err != nil { + return fmt.Errorf("updating upstream Release:\n%w", err) + } + + if err := carryStaticRenderRelease(env, opened, original, history, editor); err != nil { + return err + } + } + + var result bytes.Buffer + if err := opened.Serialize(&result); err != nil { + return fmt.Errorf("serializing prepared upstream spec:\n%w", err) + } + + if err := fileutils.WriteFile(env.FS(), path, result.Bytes(), fileperms.PublicFile); err != nil { + return fmt.Errorf("writing prepared upstream spec:\n%w", err) + } + + return nil +} + +// upstreamRenderRelease returns the final upstream 'Release' value after +// verifying that every 'Release' tag agrees on whether it uses '%autorelease'. +func upstreamRenderRelease(opened *spec.Spec) (string, error) { + releases, err := opened.GetTags("", "Release") + if err != nil { + return "", fmt.Errorf("reading upstream Release:\n%w", err) + } + + if err := checkReleaseTagsConsistent(releases); err != nil { + return "", fmt.Errorf("reading upstream Release:\n%w", err) + } + + return releases[len(releases)-1], nil +} + +// ErrMixedReleaseTags is returned when some, but not all, 'Release' tags in a +// spec use '%autorelease'. +var ErrMixedReleaseTags = errors.New("'Release' tags mix '%autorelease' and static values") + +func checkReleaseTagsConsistent(releases []string) error { + autoreleases := 0 + + for _, release := range releases { + if sources.ReleaseUsesAutorelease(release) { + autoreleases++ + } + } + + if autoreleases != 0 && autoreleases != len(releases) { + return fmt.Errorf("%d of %d 'Release' tags use '%%autorelease':\n%w", + autoreleases, len(releases), ErrMixedReleaseTags) + } + + return nil +} + +func carryLocklessInlineChangelog( + opened *spec.Spec, previous, release string, editor spec.EditorMode, +) (*spec.Spec, error) { + if previous == "" || sources.ReleaseUsesAutorelease(release) { + return opened, nil + } + + var result bytes.Buffer + if err := opened.Serialize(&result); err != nil { + return nil, fmt.Errorf("serializing upstream spec:\n%w", err) + } + + content, err := restoreRenderChangelog(result.Bytes(), []byte(previous), editor) + if err != nil { + return nil, err + } + + carried, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) + if err != nil { + return nil, fmt.Errorf("parsing spec with committed changelog:\n%w", err) + } + + return carried, nil +} + +func collectRenderUpstreamChanges( + env *azldev.Env, dir, name, version string, history lastRender, update *locklessRenderUpdate, +) error { + if history.previous == "" { + update.changes = fmt.Sprintf("Update %s to version %s", name, version) + + return nil + } + + if history.previous == history.current { + return nil + } + + if history.previousBranch != "" && history.previousBranch != history.currentBranch { + refspec := "+refs/heads/" + history.previousBranch + ":refs/remotes/origin/" + history.previousBranch + if _, err := git.RunInDir(env, env, dir, "fetch", "--no-tags", "--", "origin", refspec); err != nil { + slog.Warn("Cannot fetch last render upstream branch", + "component", name, "branch", history.previousBranch, "error", err) + } + } + + if err := resolveRenderUpstreamCommit(env, dir, history.previous); err != nil { + if env.Err() != nil { + return fmt.Errorf("resolving last render upstream commit:\n%w", env.Err()) + } + + slog.Warn("Last render upstream commit is unavailable; using current upstream pin", + "component", name, "commit", history.previous, "error", err) + update.changes = fmt.Sprintf("update to upstream %s commit %s", name, history.current) + + return nil + } + + changes, err := git.RunInDir(env, env, dir, + "log", "--oneline", "--end-of-options", history.previous+".."+history.current, "--") + if err != nil { + return fmt.Errorf("collecting upstream changes:\n%w", err) + } + + update.changes = changes + + return nil +} + +func resolveRenderUpstreamCommit(env *azldev.Env, dir, commit string) error { + if _, err := git.RunInDir(env, env, dir, + "rev-parse", "--verify", "--end-of-options", commit+"^{commit}"); err == nil { + return nil + } + + if env.Err() != nil { + return fmt.Errorf("resolving upstream commit %#q:\n%w", commit, env.Err()) + } + + if _, err := git.RunInDir(env, env, dir, "fetch", "--no-tags", "--", "origin", commit); err != nil { + slog.Warn("Cannot fetch last render upstream commit", "commit", commit, "error", err) + } + + _, err := git.RunInDir(env, env, dir, + "rev-parse", "--verify", "--end-of-options", commit+"^{commit}") + if err != nil { + return fmt.Errorf("resolving upstream commit %#q after fetching:\n%w", commit, err) + } + + return nil +} + +// releaseUpdate computes the new value of one 'Release' tag from its current value. +type releaseUpdate func(release string) (string, error) + +func keepRelease(release string) (string, error) { + return release, nil +} + +// autoreleaseBaseUpdate returns a [releaseUpdate] that applies the given +// autorelease base to each '%autorelease' 'Release' tag, preserving any other +// arguments on that tag. The base must already be validated. +func autoreleaseBaseUpdate(output string, omitDefault bool) releaseUpdate { + return func(release string) (string, error) { + if !sources.ReleaseUsesAutorelease(release) { + return release, nil + } + + return withAutoreleaseBase(release, output, omitDefault) + } +} + +func stripAutoreleaseBase(release string) (string, error) { + return autoreleaseBaseArg.ReplaceAllString(release, ""), nil +} + +func locklessRenderRelease( + env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, + versionChanged bool, +) (releaseUpdate, error) { + if !sources.ReleaseUsesAutorelease(release) || env.DryRun() { + return keepRelease, nil + } + + if history.spec != "" && (!versionChanged || renderedAutoreleaseMarker.MatchString(history.spec)) { + return carryRenderedAutorelease(env, autospec, dir, filename, history) + } + + calculated, err := autospec.CalculateBaseRelease(env, dir, filename) + if err != nil { + return nil, fmt.Errorf("calculating upstream release base:\n%w", err) + } + + if _, _, err := parseAutoreleaseBase(calculated); err != nil { + return nil, err + } + + return autoreleaseBaseUpdate(calculated, true), nil +} + +var ( + renderedAutoreleaseMarker = regexp.MustCompile(`(?m)^## RPMAUTOSPEC:[^\r\n]*autorelease`) + renderedAutoreleaseBase = regexp.MustCompile(`(?:^|\s)-b(?:\s+|=)?([^\s}]*)`) + renderedReleaseNumber = regexp.MustCompile(`^[0-9]+`) +) + +func carryRenderedAutorelease( + env *azldev.Env, autospec renderAutorelease, dir, filename string, history lastRender, +) (releaseUpdate, error) { + if !renderedAutoreleaseMarker.MatchString(history.spec) { + return carryUnprocessedAutorelease(history) + } + + expanded, err := expandCommittedRelease(env, autospec, dir, filename, history) + if err != nil { + return nil, fmt.Errorf("expanding committed Release:\n%w", err) + } + + base := renderedReleaseNumber.FindString(strings.TrimSpace(expanded)) + if _, _, err := parseAutoreleaseBase(base); err != nil { + return nil, fmt.Errorf("using committed Release %#q as autorelease base:\n%w", expanded, err) + } + + return autoreleaseBaseUpdate(base, true), nil +} + +func carryUnprocessedAutorelease(history lastRender) (releaseUpdate, error) { + previous, err := spec.OpenSpec(strings.NewReader(history.spec)) + if err != nil { + return nil, fmt.Errorf("parsing committed autorelease spec:\n%w", err) + } + + previousRelease, err := previous.GetLastTag("", "Release") + if err != nil { + return nil, fmt.Errorf("reading committed Release:\n%w", err) + } + + if !sources.ReleaseUsesAutorelease(previousRelease) { + return keepRelease, nil + } + + matches := renderedAutoreleaseBase.FindStringSubmatch(previousRelease) + if matches == nil { + return stripAutoreleaseBase, nil + } + + if _, _, err := parseAutoreleaseBase(matches[1]); err != nil { + return nil, fmt.Errorf("using committed autorelease base:\n%w", err) + } + + return autoreleaseBaseUpdate(matches[1], false), nil +} + +func prepareLocklessChangelog( + env *azldev.Env, autospec renderAutorelease, dir, filename, release string, history lastRender, +) error { + if !sources.ReleaseUsesAutorelease(release) || env.DryRun() { + return nil + } + + if history.spec != "" && !renderedAutoreleaseMarker.MatchString(history.spec) { + return carryUnprocessedChangelog(env, dir, history) + } + + var changelog string + + if history.spec == "" { + var err error + + changelog, err = autospec.GenerateChangelog(env, dir, filename) + if err != nil { + return fmt.Errorf("generating initial upstream changelog:\n%w", err) + } + } else { + previous := []byte(history.spec) + + start, end, err := renderChangelogBounds(previous, spec.EditorLegacy) + if err != nil { + return err + } + + _, changelog, _ = strings.Cut(string(previous[start:end]), "\n") + changelog = stripChangelogBoundaryComments(changelog) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(changelog), + fileperms.PublicFile); err != nil { + return fmt.Errorf("writing prepared 'changelog':\n%w", err) + } + + return nil +} + +func carryUnprocessedChangelog(env *azldev.Env, dir string, history lastRender) error { + previous, err := spec.OpenSpec(strings.NewReader(history.spec)) + if err != nil { + return fmt.Errorf("parsing committed autorelease spec:\n%w", err) + } + + release, err := previous.GetLastTag("", "Release") + if err != nil { + return fmt.Errorf("reading committed Release:\n%w", err) + } + + if !sources.ReleaseUsesAutorelease(release) { + return nil + } + + if history.tree == nil { + return errors.New("missing committed dist-git tree for 'changelog'") + } + + changelog, err := history.tree.File("changelog") + if err != nil { + return fmt.Errorf("reading committed 'changelog':\n%w", err) + } + + content, err := changelog.Contents() + if err != nil { + return fmt.Errorf("reading committed 'changelog' contents:\n%w", err) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "changelog"), []byte(content), + fileperms.PublicFile); err != nil { + return fmt.Errorf("writing committed 'changelog':\n%w", err) + } + + return nil +} + +func stripChangelogBoundaryComments(changelog string) string { + lines := strings.SplitAfter(changelog, "\n") + for idx, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + + if !strings.HasPrefix(line, "##") { + break + } + + lines[idx] = "" + } + + for idx := len(lines) - 1; idx >= 0; idx-- { + if strings.TrimSpace(lines[idx]) == "" { + continue + } + + if !strings.HasPrefix(lines[idx], "##") { + break + } + + lines[idx] = "" + } + + return strings.Join(lines, "") +} + +func expandCommittedRelease( + env *azldev.Env, autospec renderAutorelease, dir, filename string, history lastRender, +) (release string, err error) { + snapshot, err := committedRenderSnapshot(env, dir, filename, history) + if err != nil { + return "", err + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), snapshot, &err) + + release, err = autospec.ExpandRelease(env, snapshot, filename) + if err != nil { + return "", fmt.Errorf("querying committed spec:\n%w", err) + } + + return release, nil +} + +func committedRenderSnapshot( + env *azldev.Env, dir, filename string, history lastRender, +) (snapshot string, err error) { + snapshot, err = fileutils.MkdirTemp(env.FS(), filepath.Dir(dir), "azldev-render-previous-") + if err != nil { + return "", fmt.Errorf("creating committed dist-git snapshot:\n%w", err) + } + defer func(path string) { + if err != nil { + fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), path, &err) + } + }(snapshot) + + if history.tree != nil { + if err := copyTreeToFS(history.tree, env.FS(), snapshot); err != nil { + return "", fmt.Errorf("copying committed dist-git snapshot:\n%w", err) + } + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(snapshot, filename), []byte(history.spec), + fileperms.PublicFile); err != nil { + return "", fmt.Errorf("writing committed spec snapshot:\n%w", err) + } + + return snapshot, nil +} + +func locklessVersionChanged(version string, history lastRender) (bool, error) { + if history.spec == "" { + return true, nil + } + + previous, err := spec.OpenSpec(strings.NewReader(history.spec)) + if err != nil { + return false, fmt.Errorf("parsing committed dist-git spec:\n%w", err) + } + + oldVersion, err := previous.GetLastTag("", "Version") + if err != nil { + return false, fmt.Errorf("reading committed Version:\n%w", err) + } + + return oldVersion != version, nil +} + +var autoreleaseBaseArg = regexp.MustCompile(`(^|\s)-b(?:\s+|=)?[0-9]+`) + +// parseAutoreleaseBase returns the positive release number at the end of output. +func parseAutoreleaseBase(output string) (number string, value int, err error) { + fields := strings.Fields(output) + if len(fields) == 0 { + return "", 0, errors.New("empty autorelease base") + } + + number = fields[len(fields)-1] + + value, err = strconv.Atoi(number) + if err != nil || value < 1 { + return "", 0, fmt.Errorf("invalid autorelease base %#q", output) + } + + return number, value, nil +} + +func withAutoreleaseBase(release, output string, omitDefault bool) (string, error) { + number, value, err := parseAutoreleaseBase(output) + if err != nil { + return "", err + } + + if omitDefault && value == 1 { + return autoreleaseBaseArg.ReplaceAllString(release, ""), nil + } + + if autoreleaseBaseArg.MatchString(release) { + return autoreleaseBaseArg.ReplaceAllString(release, "${1}-b "+number), nil + } + + if strings.HasPrefix(release, "%{autorelease") { + depth := 0 + + for index, char := range release { + switch char { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return release[:index] + " -b " + number + release[index:], nil + } + } + } + } + + return release + " -b " + number, nil +} + +func restoreRenderChangelog(upstream, previous []byte, editor spec.EditorMode) ([]byte, error) { + oldStart, oldEnd, err := renderChangelogBounds(previous, editor) + if err != nil { + return nil, err + } + + start, end, err := renderChangelogBounds(upstream, editor) + if err != nil { + return nil, err + } + + result := append([]byte(nil), upstream[:start]...) + if start == len(upstream) && oldStart != oldEnd && !bytes.HasSuffix(result, []byte("\n")) { + result = append(result, '\n') + } + + result = append(result, previous[oldStart:oldEnd]...) + result = append(result, upstream[end:]...) + + return result, nil +} + +func renderChangelogBounds(content []byte, editor spec.EditorMode) (start int, end int, err error) { + opened, err := spec.OpenSpec(bytes.NewReader(content), spec.WithEditor(editor)) + if err != nil { + return 0, 0, fmt.Errorf("parsing spec changelog:\n%w", err) + } + + lines := bytes.SplitAfter(content, []byte("\n")) + offset := func(line int) int { + size := 0 + for _, text := range lines[:min(line, len(lines))] { + size += len(text) + } + + return size + } + start, end = len(content), len(content) + + err = opened.Visit(func(ctx *spec.Context) error { + if ctx.Target.Section == nil || ctx.Target.Section.SectName != "%changelog" { + return nil + } + + switch ctx.Target.TargetType { + case spec.SectionStartTarget: + start = offset(ctx.CurrentLineNum) + case spec.SectionEndTarget: + end = offset(ctx.CurrentLineNum) + case spec.SpecStartTarget, spec.SectionLineTarget, spec.SpecEndTarget: + } + + return nil + }) + if err != nil { + return 0, 0, fmt.Errorf("finding spec changelog:\n%w", err) + } + + return start, end, nil +} + +func finishLocklessSpec( + env *azldev.Env, path string, update *locklessRenderUpdate, changed, noChangeRebuild bool, +) (bool, error) { + if !changed { + if err := prepareLocklessRebuild(env, path, update.local, noChangeRebuild); err != nil { + return false, err + } + } + + if err := prepareRenderMessage(env, path, update); err != nil { + return false, err + } + + if update.local { + return !changed, nil + } + + bumped, err := bumpLocklessSpec(env, path, update) + + return !changed || bumped, err +} + +func prepareLocklessRebuild(env *azldev.Env, path string, local, noChangeRebuild bool) error { + autorelease, err := locklessSpecUsesAutorelease(env, path) + if err != nil { + return err + } + + if local && !autorelease { + return errLocalManualRebuild + } + + if !noChangeRebuild { + return errNoRenderChanges + } + + if !autorelease { + return nil + } + + return writeNoChangeRebuild(env, filepath.Dir(path)) +} + +func locklessSpecUsesAutorelease(env *azldev.Env, path string) (bool, error) { + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec for lockless render:\n%w", err) + } + + opened, err := spec.OpenSpec(bytes.NewReader(content)) + if err != nil { + return false, fmt.Errorf("parsing spec for lockless render:\n%w", err) + } + + release, err := opened.GetLastTag("", "Release") + if err != nil { + return false, fmt.Errorf("reading Release for lockless render:\n%w", err) + } + + return sources.ReleaseUsesAutorelease(release), nil +} + +// validateRenderNEVR rejects replacing rendered output with a lower NEVR. +// Unprocessed %autorelease specs are resolved against the component's filtered +// dist-git history, as lockless builds resolve them, instead of plain rpmspec. +func validateRenderNEVR(env *azldev.Env, preparedDir string, prep *preparedComponent) error { + existingPath := filepath.Join(prep.compOutputDir, prep.specFilename) + newPath := filepath.Join(preparedDir, prep.specFilename) + + existingHistory, err := renderSpecNeedsHistory(env, existingPath) + if err != nil { + return fmt.Errorf("%w: checking existing spec autorelease:\n%w", errRenderNEVRValidation, err) + } + + newHistory, err := renderSpecNeedsHistory(env, newPath) + if err != nil { + return fmt.Errorf("%w: checking new spec autorelease:\n%w", errRenderNEVRValidation, err) + } + + existing, rendered, err := queryRenderNEVRs(env, preparedDir, prep, existingHistory, newHistory) + if err != nil { + return err + } + + autorelease, err := locklessSpecUsesAutorelease(env, newPath) + if err != nil { + return fmt.Errorf("%w: checking new spec autorelease:\n%w", errRenderNEVRValidation, err) + } + + requirement := "higher than" + if autorelease { + requirement = "equal to or higher than" + } + + comparison := rendered.Version.Compare(&existing.Version) + if existing.Name != rendered.Name || comparison < 0 || (comparison == 0 && !autorelease) { + return fmt.Errorf("%w: new NEVR must be %s existing NEVR for the same package; "+ + "existing %#q; new %#q", errRenderNEVRValidation, requirement, + existing.Name+"-"+existing.Version.String(), rendered.Name+"-"+rendered.Version.String()) + } + + return nil +} + +func queryRenderNEVRs( + env *azldev.Env, preparedDir string, prep *preparedComponent, existingHistory, newHistory bool, +) (existing, rendered *rpm.SpecInfo, err error) { + var history renderNEVRHistory + + if existingHistory || newHistory { + history, err = openRenderNEVRHistory(env, filepath.Dir(preparedDir), prep.compOutputDir) + if err != nil { + return nil, nil, fmt.Errorf("%w: reading dist-git history:\n%w", errRenderNEVRValidation, err) + } + + defer func() { + if removeErr := history.Remove(); removeErr != nil && err == nil { + err = fmt.Errorf("%w:\n%w", errRenderNEVRValidation, removeErr) + } + }() + } + + if existingHistory { + existing, err = history.Committed(prep.specFilename) + } else { + existing, err = queryRenderSpecNEVR(env, filepath.Join(prep.compOutputDir, prep.specFilename), + prep.compOutputDir) + } + + if err != nil { + return nil, nil, fmt.Errorf("%w: querying existing NEVR:\n%w", errRenderNEVRValidation, err) + } + + if newHistory { + rendered, err = history.Rendered(preparedDir, prep.specFilename) + } else { + rendered, err = queryRenderSpecNEVR(env, filepath.Join(preparedDir, prep.specFilename), preparedDir) + } + + if err != nil { + return nil, nil, fmt.Errorf("%w: querying new NEVR:\n%w", errRenderNEVRValidation, err) + } + + return existing, rendered, nil +} + +// renderSpecNeedsHistory reports whether a spec uses %autorelease without +// having been processed by rpmautospec, so its release depends on history. +func renderSpecNeedsHistory(env *azldev.Env, path string) (bool, error) { + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec %#q:\n%w", path, err) + } + + if renderedAutoreleaseMarker.Match(content) { + return false, nil + } + + opened, err := spec.OpenSpec(bytes.NewReader(content)) + if err != nil { + return false, fmt.Errorf("parsing spec %#q:\n%w", path, err) + } + + release, err := opened.GetLastTag("", "Release") + if errors.Is(err, spec.ErrNoSuchTag) { + return false, nil + } + + if err != nil { + return false, fmt.Errorf("reading Release from %#q:\n%w", path, err) + } + + return sources.ReleaseUsesAutorelease(release), nil +} + +func queryRenderSpecNEVR(env *azldev.Env, path, dir string) (*rpm.SpecInfo, error) { + if err := prereqs.RequireExecutable(env, "rpmspec", nil); err != nil { + return nil, fmt.Errorf("lockless render NEVR validation requires 'rpmspec':\n%w", err) + } + + raw := exec.CommandContext(env, "rpmspec", "-q", "--srpm", + "--define", "_sourcedir "+dir, "--define", "_specdir "+dir, + "--queryformat", rpm.SrpmQueryFormat, path) + raw.Dir = dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := env.Command(raw) + if err != nil { + return nil, fmt.Errorf("creating NEVR query for %#q:\n%w", path, err) + } + + output, err := command.RunAndGetOutput(env) + if err != nil { + return nil, fmt.Errorf("querying NEVR for %#q (stderr: %#q):\n%w", path, stderr.String(), err) + } + + info, err := rpm.ParseSrpmQueryOutput(path, output) + if err != nil { + return nil, fmt.Errorf("parsing NEVR query for %#q:\n%w", path, err) + } + + return info, nil +} + +func writeNoChangeRebuild(env *azldev.Env, dir string) error { + if env.DryRun() { + return nil + } + + if err := prereqs.RequireExecutable(env, "date", nil); err != nil { + return fmt.Errorf("no-change rebuild requires 'date':\n%w", err) + } + + raw := exec.CommandContext(env, "date", "-Is") + raw.Dir = dir + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := env.Command(raw) + if err != nil { + return fmt.Errorf("creating rebuild timestamp command:\n%w", err) + } + + output, err := command.RunAndGetOutput(env) + if err != nil { + return fmt.Errorf("getting rebuild timestamp (%s):\n%w", stderr.String(), err) + } + + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, noChangeRebuildMarker), + []byte(output), fileperms.PublicFile); err != nil { + return fmt.Errorf("writing rebuild timestamp:\n%w", err) + } + + return nil +} + +func bumpLocklessSpec(env *azldev.Env, path string, update *locklessRenderUpdate) (changed bool, err error) { + before, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec before release bump:\n%w", err) + } + + opened, err := spec.OpenSpec(bytes.NewReader(before)) + if err != nil { + return false, fmt.Errorf("reading spec before release bump:\n%w", err) + } + + release, err := opened.GetLastTag("", "Release") + if err != nil { + return false, fmt.Errorf("reading Release before release bump:\n%w", err) + } + + if sources.ReleaseUsesAutorelease(release) { + return false, nil + } + + if err := prereqs.RequireExecutable(env, sources.RPMDevBumpspecBinary, nil); err != nil { + return false, fmt.Errorf("lockless upstream rendering requires 'rpmdev-bumpspec'; install 'rpmdevtools':\n%w", err) + } + + masked, err := maskChangelogReleaseLines(opened, before) + if err != nil { + return false, err + } + + if masked == nil { + return runLocklessBumpspec(env, path, update, before) + } + + if err := fileutils.WriteFile(env.FS(), path, masked, fileperms.PublicFile); err != nil { + return false, fmt.Errorf("masking %%changelog Release lines before release bump:\n%w", err) + } + + changed, err = runLocklessBumpspec(env, path, update, before) + if err != nil { + if restoreErr := fileutils.WriteFile(env.FS(), path, before, fileperms.PublicFile); restoreErr != nil { + err = errors.Join(err, fmt.Errorf("restoring spec after failed release bump:\n%w", restoreErr)) + } + + return false, err + } + + return changed, nil +} + +// changelogReleaseMask prefixes '%changelog' lines that rpmdev-bumpspec would otherwise +// treat as Release tags; rpmdev-bumpspec does not distinguish spec sections. +const changelogReleaseMask = "azldev-masked-changelog-line:" + +var changelogReleaseLinePattern = regexp.MustCompile(`(?i)^release\s*:`) + +// maskChangelogReleaseLines returns the spec with '%changelog' lines starting with 'Release:' +// masked, or nil when there are no such lines. +func maskChangelogReleaseLines(opened *spec.Spec, original []byte) ([]byte, error) { + masked := false + + err := opened.Visit(func(ctx *spec.Context) error { + if ctx.Target.TargetType != spec.SectionLineTarget || ctx.RawLine == nil || + ctx.CurrentSection.SectType != spec.ChangelogSection || + !changelogReleaseLinePattern.MatchString(*ctx.RawLine) { + return nil + } + + ctx.ReplaceLine(changelogReleaseMask + *ctx.RawLine) + + masked = true + + return nil + }) + if err != nil { + return nil, fmt.Errorf("finding %%changelog Release lines:\n%w", err) + } + + if !masked { + return nil, nil + } + + if bytes.Contains(original, []byte(changelogReleaseMask)) { + return nil, fmt.Errorf("spec already contains reserved text %#q", changelogReleaseMask) + } + + var buffer bytes.Buffer + if err := opened.Serialize(&buffer); err != nil { + return nil, fmt.Errorf("masking %%changelog Release lines:\n%w", err) + } + + return buffer.Bytes(), nil +} + +// runLocklessBumpspec runs rpmdev-bumpspec, unmasks any masked '%changelog' lines, and +// reports whether the spec differs from before. +func runLocklessBumpspec( + env *azldev.Env, path string, update *locklessRenderUpdate, before []byte, +) (changed bool, err error) { + raw := exec.CommandContext(env, sources.RPMDevBumpspecBinary, "-c", update.subject, path) + raw.Dir = filepath.Dir(path) + + wrapperDir, err := prepareRenderBumpspecEnvironment(env, raw, path) + if wrapperDir != "" { + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), wrapperDir, &err) + } + + if err != nil { + return false, err + } + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + command, err := env.Command(raw) + if err != nil { + return false, fmt.Errorf("creating render release bump command:\n%w", err) + } + + if err := command.Run(env); err != nil { + return false, fmt.Errorf("bumping render release (%s):\n%w", stderr.String(), err) + } + + after, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return false, fmt.Errorf("reading spec after release bump:\n%w", err) + } + + if bytes.Contains(after, []byte(changelogReleaseMask)) { + after = bytes.ReplaceAll(after, []byte(changelogReleaseMask), nil) + if err := fileutils.WriteFile(env.FS(), path, after, fileperms.PublicFile); err != nil { + return false, fmt.Errorf("unmasking %%changelog Release lines after release bump:\n%w", err) + } + } + + return !bytes.Equal(before, after), nil +} + +func prepareRenderBumpspecEnvironment(env *azldev.Env, raw *exec.Cmd, specPath string) (string, error) { + dir, err := fileutils.MkdirTempInTempDir(env.FS(), "azldev-render-rpm-") + if err != nil { + return "", fmt.Errorf("creating temporary render RPM wrapper directory:\n%w", err) + } + + if strings.ContainsRune(dir, os.PathListSeparator) { + return dir, fmt.Errorf("temporary RPM wrapper directory %#q contains the PATH separator", dir) + } + + wrapper := sources.GenerateRPMDevBumpspecWrapper(sources.RPMDevBumpspecRequest{SpecPath: specPath}) + if err := fileutils.WriteFile(env.FS(), filepath.Join(dir, "rpm"), + []byte(wrapper), fileperms.PublicExecutable); err != nil { + return dir, fmt.Errorf("writing temporary render RPM wrapper:\n%w", err) + } + + originalPath := env.OSEnv().Getenv("PATH") + raw.Env = append(raw.Environ(), + "PATH="+dir+string(os.PathListSeparator)+originalPath, + "RPMDEV_BUMPSPEC_ORIGINAL_PATH="+originalPath) + + return dir, nil +} + +func commitLocklessRenders( + env *azldev.Env, prepared []*preparedComponent, results []*RenderResult, checkOnly bool, +) { + if checkOnly || env.DryRun() { + return + } + + // Serialize writes to the project index. + includeInputs := true + + for _, prep := range prepared { + result := results[prep.index] + + if prep.update != nil && !prep.update.noGitCommit && result.Status == renderStatusOK { + committed, err := commitLocklessRender(env, prep.update, includeInputs) + if err != nil { + result.Status = renderStatusError + result.Error = err.Error() + } else if committed { + includeInputs = false + } + } + + if !prep.lockless { + continue + } + + if result.Status != renderStatusOK { + if prep.outputTouched { + if err := restoreLocklessOutput(env, prep); err != nil { + result.Error += "\n" + err.Error() + } + } + } else { + writeRenderedAlias(env, prep) + } + } +} + +func restoreLocklessOutput(env *azldev.Env, prep *preparedComponent) error { + if err := env.FS().RemoveAll(prep.compOutputDir); err != nil { + return fmt.Errorf("clearing failed rendered output %#q:\n%w", prep.compOutputDir, err) + } + + root := env.ProjectDir() + if prep.update != nil { + root = prep.update.repoRoot + } + + path, err := filepath.Rel(root, prep.compOutputDir) + if err != nil { + return fmt.Errorf("locating rendered output in the project repository:\n%w", err) + } + + path = filepath.ToSlash(path) + ctx := context.WithoutCancel(env) + + // Reset this path first so checkout cannot reuse files staged by the failed render. + if _, err := git.RunInDir(ctx, env, root, "reset", "--quiet", "--", path); err != nil { + return fmt.Errorf("resetting failed rendered output in the index:\n%w", err) + } + + if prep.outputExisted { + if _, err := git.RunInDir(ctx, env, root, "checkout", "HEAD", "--", path); err != nil { + return fmt.Errorf("restoring committed rendered output %#q:\n%w", prep.compOutputDir, err) + } + } + + return nil +} + +func commitLocklessRender(env *azldev.Env, update *locklessRenderUpdate, includeInputs bool) (bool, error) { + paths := []string{update.path} + if includeInputs { + inputs, err := stageLocklessRenderInputs(env, update) + if err != nil { + return false, err + } + + paths = append(paths, inputs...) + } + + if _, err := git.RunInDir(env, env, update.repoRoot, "add", "--all", "--", update.path); err != nil { + return false, fmt.Errorf("staging rendered component:\n%w", err) + } + + diffArgs := append([]string{"diff", "--cached", "--name-only", "--"}, paths...) + + diff, err := git.RunInDir(env, env, update.repoRoot, diffArgs...) + if err != nil { + return false, fmt.Errorf("checking staged render:\n%w", err) + } + + if diff == "" { + return false, nil + } + + if err := runLocklessRenderCommit(env, update, paths); err != nil { + return false, fmt.Errorf("committing rendered component:\n%w", err) + } + + return true, nil +} + +func stageLocklessRenderInputs(env *azldev.Env, update *locklessRenderUpdate) ([]string, error) { + renderedPath := filepath.ToSlash(filepath.Dir(filepath.Dir(update.path))) + exclusion := ":(top,exclude,literal)" + renderedPath + + if _, err := git.RunInDir(env, env, update.repoRoot, "add", "--all", "--", ".", exclusion); err != nil { + return nil, fmt.Errorf("staging uncommitted files outside rendered specs:\n%w", err) + } + + status, err := git.RunInDir(env, env, update.repoRoot, + "diff", "--cached", "--name-status", "--no-renames", "-z", "--", ".", exclusion) + if err != nil { + return nil, fmt.Errorf("checking staged files outside rendered specs:\n%w", err) + } + + // With '--no-renames', NUL-delimited name/status output contains status/path pairs. + fields := strings.Split(status, "\x00") + + var paths []string + + for index := 1; index < len(fields); index += 2 { + paths = append(paths, ":(top,literal)"+fields[index]) + } + + return paths, nil +} diff --git a/internal/app/azldev/cmds/component/render_lockless_internal_test.go b/internal/app/azldev/cmds/component/render_lockless_internal_test.go new file mode 100644 index 000000000..f9b6bb730 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_lockless_internal_test.go @@ -0,0 +1,2221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "testing" + + gogit "github.com/go-git/go-git/v5" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +const ( + renderTestRevParse = "rev-parse" + renderTestProjectRoot = "/project" + renderTestWrapperCleanupFailure = "cleanup" +) + +func renderTestSpec(version, release, changelog string) string { + return fmt.Sprintf("Name: curl\nVersion: %s\nRelease: %s\nSummary: Test\nLicense: MIT\n"+ + "\n%%description\nTest\n\n%%changelog\n%s", version, release, changelog) +} + +func TestReadLastRender_CommittedConfigAndSpec(t *testing.T) { + root := []byte(`includes = ["component.toml", "pins/*.toml"] +[project] +default-distro = { name = "test", version = "3" } +rendered-specs-dir = "SPECS" +[distros.test.versions."1"] +release-ver = "1" +dist-git-branch = "f40" +[distros.test.versions."2"] +release-ver = "2" +dist-git-branch = "f41" +[distros.test.versions."3"] +release-ver = "3" +dist-git-branch = "f42" +`) + oldSpec := renderTestSpec("1", "9%{?dist}", "* Previous\n- Keep exactly\n") + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: map[string][]byte{ + "azldev.toml": root, + "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n" + + "upstream-distro = { name = 'test', version = '1' }\n"), + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'aaaaaaa'\n"), + "SPECS/c/curl/curl.spec": []byte(oldSpec), + }}, + {files: map[string][]byte{ + "pins/curl.toml": []byte("[components.curl.spec]\nupstream-commit = 'bbbbbbb'\n"), + "component.toml": []byte("[components.curl.spec]\ntype = 'upstream'\n" + + "upstream-distro = { name = 'test', version = '2' }\n"), + }}, + {files: map[string][]byte{"README": []byte("unrelated later commit")}}, + }) + worktree, err := repo.Worktree() + require.NoError(t, err) + dirty, err := worktree.Filesystem.Create("SPECS/c/curl/curl.spec") + require.NoError(t, err) + _, err = dirty.Write([]byte("uncommitted spec must not be used")) + require.NoError(t, err) + require.NoError(t, dirty.Close()) + + env := testutils.NewTestEnv(t) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[2], "--", "SPECS/c/curl/", + }, cmd.Args) + + return hashes[0] + "\n", nil + } + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl", false) + require.NoError(t, err) + assert.Equal(t, oldSpec, history.spec) + require.NotNil(t, history.tree) + committed, err := history.tree.File("curl.spec") + require.NoError(t, err) + content, err := committed.Contents() + require.NoError(t, err) + assert.Equal(t, oldSpec, content) + assert.Equal(t, "aaaaaaa", history.previous) + assert.Equal(t, "bbbbbbb", history.current) + assert.Equal(t, "f40", history.previousBranch) + assert.Equal(t, "f41", history.currentBranch) +} + +func TestReadLastRender_NativeHistoryErrors(t *testing.T) { + const renderPath = "SPECS/c/curl[1]" + + logErr := errors.New("git log failed") + + for _, testCase := range []struct { + name, output, wantError string + commandErr error + }{ + {name: "no matching commit"}, + {name: "git failure", commandErr: logErr, wantError: "finding last render commit"}, + {name: "unknown commit", output: strings.Repeat("a", 40), wantError: "reading last render commit"}, + } { + t.Run(testCase.name, func(t *testing.T) { + previous := renderTestSpec("1", "9", "old\n") + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + renderPath + "/curl.spec": []byte(previous), + }}}) + env := testutils.NewTestEnv(t) + called := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + called = true + + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", renderPath + "/", + }, cmd.Args) + + return testCase.output, testCase.commandErr + } + + history, err := readLastRender(env.Env, repo, "/project", renderPath, "curl", false) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.NoError(t, err) + } + + if testCase.commandErr != nil { + require.ErrorIs(t, err, testCase.commandErr) + } + + assert.True(t, called) + assert.Equal(t, previous, history.spec) + assert.Empty(t, history.previous) + assert.Empty(t, history.current) + }) + } +} + +func TestReadLastRender_HistoricalLockFallback(t *testing.T) { + const ( + oldLock = "version = 1\nupstream-commit = 'aaaaaaa'\n" + config = `[project] +default-distro = { name = "test", version = "1" } +lock-dir = "../historical-locks" +[distros.test.versions."1"] +release-ver = "1" +dist-git-branch = "f40" +` + ) + for _, testCase := range []struct { + name, selection, lock, pin, want string + noSelection, noLock, noLockDir bool + missingCurrent bool + }{ + {name: "historically locked", selection: "rust-asyncband\n", lock: oldLock, want: "aaaaaaa"}, + {name: "empty selection", lock: oldLock, want: "aaaaaaa"}, + {name: "exact names", selection: "Curl\ncurl-extra\n", lock: oldLock, want: "aaaaaaa"}, + {name: "listed component", selection: " curl \r\n", lock: oldLock}, + {name: "missing selection", noSelection: true, lock: oldLock, want: "aaaaaaa"}, + {name: "missing selection and lock", noSelection: true, noLock: true}, + {name: "missing lock", noLock: true}, + {name: "corrupt lock", lock: "invalid = ["}, + {name: "empty lock pin", lock: "version = 1\n"}, + {name: "no configured lock directory", noLockDir: true, lock: oldLock}, + {name: "configured pin takes precedence", pin: "ccccccc", lock: oldLock, want: "ccccccc"}, + {name: "no fallback for current pin", lock: oldLock, missingCurrent: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + oldConfig := config + if testCase.noLockDir { + oldConfig = strings.ReplaceAll(oldConfig, `lock-dir = "../historical-locks"`, `lock-dir = ""`) + } + + component := "[components.curl.spec]\ntype = 'upstream'\n" + + files := map[string][]byte{ + "azldev.toml": []byte(`includes = ["settings/project.toml", "components.toml"]`), + "settings/project.toml": []byte(oldConfig), + "components.toml": []byte(component + + "upstream-commit = '" + testCase.pin + "'\n"), + "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", "old\n")), + } + if !testCase.noSelection { + files["lockless-components.txt"] = []byte(testCase.selection) + } + + if !testCase.noLock { + files["historical-locks/curl.lock"] = []byte(testCase.lock) + } + + current := component + selection := "rust-asyncband\n" + + if !testCase.missingCurrent { + current += "upstream-commit = 'bbbbbbb'\n" + selection = "curl\n" + } + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{ + {files: files}, + {files: map[string][]byte{ + "components.toml": []byte(current), + "settings/project.toml": []byte(strings.ReplaceAll( + strings.ReplaceAll(config, "historical-locks", "current-locks"), "f40", "f41")), + "lockless-components.txt": []byte(selection), + "historical-locks/curl.lock": []byte("version = 1\nupstream-commit = 'changed-old-lock'\n"), + "current-locks/curl.lock": []byte("version = 1\nupstream-commit = 'current-lock'\n"), + }}, + }) + worktree, err := repo.Worktree() + require.NoError(t, err) + dirty, err := worktree.Filesystem.Create("historical-locks/curl.lock") + require.NoError(t, err) + _, err = dirty.Write([]byte("uncommitted lock")) + require.NoError(t, err) + require.NoError(t, dirty.Close()) + env := testutils.NewTestEnv(t) + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return hashes[0], nil + } + + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl", false) + if testCase.missingCurrent { + require.ErrorContains(t, err, "current render upstream commit") + + return + } + + require.NoError(t, err) + assert.Equal(t, testCase.want, history.previous) + assert.Equal(t, "bbbbbbb", history.current) + assert.Equal(t, "f41", history.currentBranch) + + if testCase.want != "" { + assert.Equal(t, "f40", history.previousBranch) + } + }) + } +} + +func TestCollectRenderUpstreamChanges_Branches(t *testing.T) { + for _, testCase := range []struct { + name, previous, current, oldBranch, newBranch string + wantFetch, fetchFails, wantLog bool + }{ + { + name: "same branch", previous: "old", current: "new", + oldBranch: "f41", newBranch: "f41", wantLog: true, + }, + { + name: "branch change", previous: "old", current: "new", + oldBranch: "f40", newBranch: "f41", wantFetch: true, wantLog: true, + }, + { + name: "branch fetch fails", previous: "old", current: "new", + oldBranch: "f40", newBranch: "f41", wantFetch: true, fetchFails: true, wantLog: true, + }, + { + name: "same commit on different branch", previous: "same", current: "same", + oldBranch: "f40", newBranch: "f41", + }, + {name: "initial render", current: "new", newBranch: "f41"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + fetchErr := errors.New("fetch failed") + + var fetched, logged bool + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == renderTestRevParse { + assert.Equal(t, []string{ + "git", "-C", "/working", renderTestRevParse, "--verify", "--end-of-options", "old^{commit}", + }, cmd.Args) + + return "old\n", nil + } + + if cmd.Args[3] == "fetch" { + fetched = true + + assert.False(t, logged, "fetch must run before comparing commits") + assert.Equal(t, []string{ + "git", "-C", "/working", "fetch", "--no-tags", "--", "origin", + "+refs/heads/f40:refs/remotes/origin/f40", + }, cmd.Args) + + if testCase.fetchFails { + return "", fetchErr + } + + return "", nil + } + + logged = true + + assert.Equal(t, []string{ + "git", "-C", "/working", "log", "--oneline", "--end-of-options", "old..new", "--", + }, cmd.Args) + + return "abc123 Upstream change\n", nil + } + update := &locklessRenderUpdate{} + + err := collectRenderUpstreamChanges(env.Env, "/working", "curl", "1", + lastRender{ + previous: testCase.previous, current: testCase.current, + previousBranch: testCase.oldBranch, currentBranch: testCase.newBranch, + }, update) + require.NoError(t, err) + + assert.Equal(t, testCase.wantFetch, fetched) + assert.Equal(t, testCase.wantLog, logged) + + if testCase.wantLog { + assert.Equal(t, "abc123 Upstream change", update.changes) + } + }) + } +} + +func TestCollectRenderUpstreamChanges_MissingCommit(t *testing.T) { + for _, testCase := range []struct { + name string + present, recovered, fetchFail bool + logFail, cancelResolve bool + branchFail bool + }{ + {name: "already available", present: true}, + {name: "fetched commit", recovered: true}, + {name: "failed fetch", fetchFail: true}, + {name: "failed fetch still recovers commit", recovered: true, fetchFail: true}, + {name: "successful fetch without commit"}, + {name: "failed branch and commit fetch", branchFail: true, fetchFail: true}, + {name: "commit recovered after branch fetch failure", branchFail: true, recovered: true}, + {name: "log failure is fatal", present: true, logFail: true}, + {name: "cancelled resolution", cancelResolve: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + workerEnv, cancel := env.Env.WithCancel() + defer cancel() + + const ( + previous = "1111111111111111111111111111111111111111" + current = "2222222222222222222222222222222222222222" + ) + + logErr := errors.New("log failed") + + var ( + commands []string + resolutions int + ) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args[3]) + + switch cmd.Args[3] { + case renderTestRevParse: + resolutions++ + + assert.Equal(t, []string{ + "git", "-C", "/working", renderTestRevParse, "--verify", "--end-of-options", previous + "^{commit}", + }, cmd.Args) + + if testCase.cancelResolve { + cancel() + + return "", context.Canceled + } + + if testCase.present || (resolutions == 2 && testCase.recovered) { + return previous, nil + } + + return "", errors.New("unknown revision") + case "fetch": + if strings.HasPrefix(cmd.Args[7], "+refs/heads/") { + assert.Equal(t, "+refs/heads/f40:refs/remotes/origin/f40", cmd.Args[7]) + + return "", errors.New("branch unavailable") + } + + assert.Equal(t, []string{ + "git", "-C", "/working", "fetch", "--no-tags", "--", "origin", previous, + }, cmd.Args) + + if testCase.fetchFail { + return "", errors.New("fetch failed") + } + + return "", nil + case "log": + assert.Equal(t, []string{ + "git", "-C", "/working", "log", "--oneline", "--end-of-options", previous + ".." + current, "--", + }, cmd.Args) + + if testCase.logFail { + return "", logErr + } + + return "def456 Fetched upstream change\n", nil + default: + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } + } + + update := &locklessRenderUpdate{} + + history := lastRender{previous: previous, current: current} + if testCase.branchFail { + history.previousBranch, history.currentBranch = "f40", "f41" + } + + err := collectRenderUpstreamChanges(workerEnv, "/working", "rust-asyncband", "1", + history, update) + + switch { + case testCase.cancelResolve: + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, []string{renderTestRevParse}, commands) + assert.Empty(t, update.changes) + case testCase.logFail: + require.ErrorIs(t, err, logErr) + require.ErrorContains(t, err, "collecting upstream changes") + assert.Equal(t, []string{renderTestRevParse, "log"}, commands) + default: + require.NoError(t, err) + + expected := []string{renderTestRevParse} + if testCase.branchFail { + expected = append([]string{"fetch"}, expected...) + } + + if !testCase.present { + expected = append(expected, "fetch", renderTestRevParse) + } + + if testCase.present || testCase.recovered { + expected = append(expected, "log") + + assert.Equal(t, "def456 Fetched upstream change", update.changes) + } else { + assert.Equal(t, "update to upstream rust-asyncband commit "+current, update.changes) + } + + assert.Equal(t, expected, commands) + } + }) + } +} + +func TestReadLastRender_AbsentAndInvalidHistory(t *testing.T) { + valid := []byte(`[project] +default-distro = { name = "test", version = "1" } +[distros.test.versions."1"] +release-ver = "1" +[components.curl.spec] +type = "upstream" +upstream-commit = "bbbbbbb" +`) + for _, testCase := range []struct { + name string + oldConfig []byte + oldSpec bool + current []byte + wantCurrent string + wantError string + }{ + {name: "no committed directory", oldConfig: valid}, + { + name: "invalid previous config", oldSpec: true, oldConfig: []byte("invalid = ["), + current: valid, wantCurrent: "bbbbbbb", + }, + { + name: "invalid current config", oldSpec: true, oldConfig: valid, + current: []byte("invalid = ["), wantError: "current render upstream commit", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + files := map[string][]byte{"azldev.toml": testCase.oldConfig} + if testCase.oldSpec { + files["SPECS/c/curl/curl.spec"] = []byte(renderTestSpec("1", "1", "old\n")) + } + + commits := []testRepoCommit{{files: files}} + if testCase.current != nil { + commits = append(commits, testRepoCommit{files: map[string][]byte{"azldev.toml": testCase.current}}) + } + + repo, hashes := testRepoWithCommits(t, commits) + env := testutils.NewTestEnv(t) + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return hashes[0], nil + } + + history, err := readLastRender(env.Env, repo, "/project", "SPECS/c/curl", "curl", false) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + assert.Empty(t, history.previous) + assert.Equal(t, testCase.wantCurrent, history.current) + }) + } +} + +type renderAutospecStub struct { + calls []string + err error + base string + expand func(string, string) (string, error) +} + +func (s *renderAutospecStub) GenerateChangelog(_ context.Context, _, _ string) (string, error) { + s.calls = append(s.calls, "changelog") + + return "* Full upstream history\n- Original change\n", s.err +} + +func (s *renderAutospecStub) CalculateBaseRelease(_ context.Context, _, _ string) (string, error) { + s.calls = append(s.calls, "release") + + if s.base != "" { + return s.base, s.err + } + + return "Calculated release number: 42\n", s.err +} + +func (s *renderAutospecStub) CalculateRelease(_ context.Context, dir, filename string) (string, error) { + s.calls = append(s.calls, "complete-release") + + return s.expand(dir, filename) +} + +func (s *renderAutospecStub) ExpandRelease(_ context.Context, dir, filename string) (string, error) { + s.calls = append(s.calls, "expand") + + return s.expand(dir, filename) +} + +func TestPrepareLocklessSpec_ReleaseChangelogAndUpstreamChanges(t *testing.T) { + const oldChangelog = "* Previous downstream entry\n- Preserve spaces \n" + for _, testCase := range []struct { + name string + version string + release string + history lastRender + wantRelease string + wantChanges string + bootstrap bool + base string + }{ + { + name: "same upstream", version: "1", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", + }, + wantRelease: "9%{?dist}", + }, + { + name: "upstream changes without version change", version: "1", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", + }, + wantRelease: "9%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + }, + { + name: "version change keeps upstream release", version: "2", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "new", + }, + wantRelease: "1%{?dist}", wantChanges: "abc123 Upstream fix\ndef456 Another fix", + }, + { + name: "version change without upstream changes", version: "2", release: "1%{?dist}", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", + }, + wantRelease: "1%{?dist}", + }, + { + name: "initial static spec", version: "2", release: "3%{?dist}", + wantRelease: "3%{?dist}", wantChanges: "Update curl to version 2", + }, + { + name: "initial autorelease", version: "2", release: "%autorelease -b 7", + wantRelease: "%autorelease -b 42", wantChanges: "Update curl to version 2", bootstrap: true, + }, + { + name: "initial default base", version: "2", release: "%autorelease -b 7 -e test", + wantRelease: "%autorelease -e test", wantChanges: "Update curl to version 2", + bootstrap: true, base: "Calculated release number: 1\n", + }, + { + name: "existing autorelease", version: "1", release: "%autorelease", + history: lastRender{ + spec: renderTestSpec("1", "9%{?dist}", oldChangelog), previous: "old", current: "old", + }, + wantRelease: "%autorelease", + }, + { + name: "unknown previous upstream", version: "1", release: "1", + history: lastRender{spec: renderTestSpec("1", "9", oldChangelog), current: "new"}, + wantRelease: "9", wantChanges: "Update curl to version 1", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec(testCase.version, testCase.release, "* Upstream\n- New entry\n")), + fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", []byte("stale"), + fileperms.PublicFile)) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == renderTestRevParse { + return "old\n", nil + } + + assert.Equal(t, []string{ + "git", "-C", "/working", "log", "--oneline", "--end-of-options", + "old..new", "--", + }, cmd.Args) + + return "abc123 Upstream fix\ndef456 Another fix\n", nil + } + called := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.False(t, called, "only the final release bump should run") + called = true + + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", "update curl", path}, cmd.Args) + + return nil + } + stub := &renderAutospecStub{base: testCase.base} + update := &locklessRenderUpdate{subject: "update curl"} + require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", + testCase.version, testCase.history, update)) + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + testCase.history) + require.NoError(t, err) + assert.False(t, called, "carryover must not perform the final bump") + + prepared, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(prepared), "Release: "+testCase.wantRelease+"\n") + + if testCase.history.spec != "" && testCase.release != "%autorelease" { + assert.True(t, strings.HasSuffix(string(prepared), oldChangelog)) + } else { + assert.True(t, strings.HasSuffix(string(prepared), "* Upstream\n- New entry\n")) + } + + _, err = bumpLocklessSpec(env.Env, path, update) + require.NoError(t, err) + assert.Equal(t, !sources.ReleaseUsesAutorelease(testCase.wantRelease), called) + assert.Equal(t, testCase.wantChanges, update.changes) + + if testCase.bootstrap { + assert.Equal(t, []string{"changelog", "release"}, stub.calls) + + changelog, err := fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, "* Full upstream history\n- Original change\n", string(changelog)) + } else { + assert.Empty(t, stub.calls) + } + }) + } +} + +const carriedAutoreleaseChangelog = "* Mon Nov 02 2026 Packager - 1-9\n- Preserve spaces \n\n" + +func testCommittedAutoreleaseHistory(t *testing.T) lastRender { + t.Helper() + + content := "## RPMAUTOSPEC: processed with autorelease, autochangelog\n%global carried_release 9\n" + + renderTestSpec("1", "%{carried_release}%{?dist}", + "## START: rpmautospec generated changelog\n"+carriedAutoreleaseChangelog+ + "## END: rpmautospec generated changelog\n") + + "%files\n/file\n" + repo, commits := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "curl.spec": []byte(content), "macros.inc": []byte("committed macro definitions"), + }}}) + tree, err := resolveTree(repo, commits[0]) + require.NoError(t, err) + + return lastRender{spec: content, tree: tree, previous: "old", current: "old"} +} + +func TestStripChangelogBoundaryComments(t *testing.T) { + for _, testCase := range []struct { + name, input, want string + }{ + {name: "empty"}, + {name: "unchanged", input: "* Entry\n- Keep spaces \n\n", want: "* Entry\n- Keep spaces \n\n"}, + {name: "leading only", input: "## First\n## Second\n* Entry\n", want: "* Entry\n"}, + {name: "trailing only", input: "* Entry\n## First\n## Second", want: "* Entry\n"}, + { + name: "boundary blocks and interior comments", + input: "\n## Start\n\n* Entry\n## Interior\n- Keep\n\n## End\n\n", + want: "\n\n* Entry\n## Interior\n- Keep\n\n\n", + }, + {name: "comments only", input: "## Start\n## End\n", want: ""}, + {name: "CRLF", input: "## Start\r\n* Entry\r\n## End\r\n", want: "* Entry\r\n"}, + { + name: "other comments remain", + input: "# Start\n## Interior\n* Entry\n ## Indented\n", + want: "# Start\n## Interior\n* Entry\n ## Indented\n", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + assert.Equal(t, testCase.want, stripChangelogBoundaryComments(testCase.input)) + }) + } +} + +func TestPrepareLocklessSpec_CommittedAutorelease(t *testing.T) { + for _, testCase := range []struct { + name string + version string + upstreamChanged bool + wantRelease string + expanded string + }{ + {name: "same upstream", version: "1", wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43"}, + { + name: "new upstream same version", version: "1", upstreamChanged: true, + wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43", + }, + { + name: "version change", version: "2", upstreamChanged: true, + wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43", + }, + { + name: "version change without upstream changes", version: "2", + wantRelease: "%autorelease -b 9 -e test", expanded: "9.fc43", + }, + { + name: "default expanded base", version: "2", + wantRelease: "%autorelease -e test", expanded: "1.fc43", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + history := testCommittedAutoreleaseHistory(t) + if testCase.upstreamChanged { + history.current = "new" + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec(testCase.version, "%autorelease -b 2 -e test", "%autochangelog\n")), + fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", []byte("stale"), + fileperms.PublicFile)) + + var snapshot string + + stub := &renderAutospecStub{expand: func(dir, filename string) (string, error) { + snapshot = dir + changelog, err := fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, carriedAutoreleaseChangelog, string(changelog), + "carry the changelog before expanding the committed release") + + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(dir, filename)) + require.NoError(t, err) + assert.Equal(t, history.spec, string(content)) + content, err = fileutils.ReadFile(env.TestFS, filepath.Join(dir, "macros.inc")) + require.NoError(t, err) + assert.Equal(t, "committed macro definitions", string(content)) + + return " " + testCase.expanded + "\n", nil + }} + + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "abc123 Upstream fix\n", nil + } + bumped := false + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + bumped = true + + return nil + } + update := &locklessRenderUpdate{subject: "updated curl"} + require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", + testCase.version, history, update)) + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history) + require.NoError(t, err) + assert.False(t, bumped) + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+testCase.wantRelease+"\n") + assert.Contains(t, string(content), "%changelog\n%autochangelog\n") + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, carriedAutoreleaseChangelog, string(content)) + + _, err = bumpLocklessSpec(env.Env, "/working/curl.spec", update) + require.NoError(t, err) + assert.False(t, bumped) + + assert.Equal(t, []string{"expand"}, stub.calls) + assert.NotEqual(t, "/working", snapshot) + exists, err := fileutils.DirExists(env.TestFS, snapshot) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestPrepareLocklessSpec_UnprocessedAutorelease(t *testing.T) { + const changelog = "## Keep committed comments\r\n* Entry\r\n- Keep spaces \r\n\r\n" + + for _, testCase := range []struct { + name, previous, working, want string + newVersion bool + noWorkingChangelog bool + base string + }{ + { + name: "changed version recalculates base", previous: "%autorelease -b 17", + working: "%autorelease -b 2 -e test", want: "%autorelease -b 42 -e test", + newVersion: true, + }, + { + name: "changed version omits default base", previous: "%autorelease -b 17", + working: "%autorelease -b 2 -e test", want: "%autorelease -e test", + newVersion: true, base: "Calculated release number: 1\n", + }, + { + name: "same version retains explicit committed default", previous: "%autorelease -b 1", + working: "%autorelease -b 2", want: "%autorelease -b 1", + }, + { + name: "changed version recalculates braced base", previous: "%{autorelease -b17}", + working: "%{autorelease -b 2 -e %{suffix}}", want: "%{autorelease -b 42 -e %{suffix}}", + newVersion: true, noWorkingChangelog: true, + }, + { + name: "braced compact base", previous: "%{autorelease -b17}", + working: "%{autorelease -b 2 -e %{suffix}}", want: "%{autorelease -b 17 -e %{suffix}}", + noWorkingChangelog: true, + }, + { + name: "equals base", previous: "%autorelease -b=17", + working: "%autorelease", want: "%autorelease -b 17", + }, + { + name: "omitted base", previous: "%autorelease", + working: "%autorelease -b 2 -e test", want: "%autorelease -e test", + }, + { + name: "omitted braced base", previous: "%{autorelease}", + working: "%{autorelease -e %{suffix} -b2}", want: "%{autorelease -e %{suffix}}", + }, + { + name: "both bases omitted", previous: "%autorelease", + working: "%autorelease", want: "%autorelease", + }, + { + name: "same version carries release base", previous: "%autorelease -b 17", + working: "%autorelease -b 2", want: "%autorelease -b 17", + }, + { + name: "changed version calculates base despite prior omission", previous: "%autorelease", + working: "%autorelease -b 2", want: "%autorelease -b 42", + newVersion: true, + }, + { + name: "changed version adds omitted working base", previous: "%autorelease -b 17", + working: "%autorelease", want: "%autorelease -b 42", + newVersion: true, + }, + { + name: "changed version ignores old invalid base", previous: "%autorelease -b invalid", + working: "%autorelease -b 2", want: "%autorelease -b 42", + newVersion: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + committed := renderTestSpec("1", testCase.previous, "%autochangelog\n") + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "curl.spec": []byte(committed), "changelog": []byte(changelog), + }}}) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + env := testutils.NewTestEnv(t) + + version := "1" + if testCase.newVersion { + version = "2" + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec(version, testCase.working, "%autochangelog\n")), fileperms.PublicFile)) + + if !testCase.noWorkingChangelog { + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("replace upstream changelog"), fileperms.PublicFile)) + } + + stub := &renderAutospecStub{base: testCase.base} + err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + lastRender{spec: committed, tree: tree}) + require.NoError(t, err) + + if testCase.newVersion { + assert.Equal(t, []string{"release"}, stub.calls) + } else { + assert.Empty(t, stub.calls) + } + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+testCase.want+"\n") + assert.Contains(t, string(content), "%changelog\n%autochangelog\n") + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, changelog, string(content)) + }) + } +} + +func TestPrepareLocklessSpec_UnprocessedAutoreleaseFailure(t *testing.T) { + for _, testCase := range []struct { + name, release, want string + missingFile bool + missingTree bool + newVersion bool + releaseError error + }{ + { + name: "missing changelog", release: "%autorelease -b 17", + missingFile: true, want: "reading committed 'changelog'", + }, + { + name: "missing tree", release: "%autorelease -b 17", + missingTree: true, want: "missing committed dist-git tree", + }, + { + name: "zero base", release: "%autorelease -b 0", + want: "using committed autorelease base", + }, + { + name: "invalid base", release: "%autorelease -b invalid", + want: "using committed autorelease base", + }, + { + name: "changed version query failure", release: "%autorelease -b 17", + newVersion: true, releaseError: errors.New("query failed"), + want: "calculating upstream release base", + }, + { + name: "changed version missing changelog", release: "%autorelease -b 17", + newVersion: true, missingFile: true, want: "reading committed 'changelog'", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + committed := renderTestSpec("1", testCase.release, "%autochangelog\n") + + files := map[string][]byte{"curl.spec": []byte(committed)} + if !testCase.missingFile { + files["changelog"] = []byte("committed changelog") + } + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: files}}) + tree, err := resolveTree(repo, hashes[0]) + require.NoError(t, err) + + if testCase.missingTree { + tree = nil + } + + env := testutils.NewTestEnv(t) + + version := "1" + if testCase.newVersion { + version = "2" + } + + original := renderTestSpec(version, "%autorelease -b 2", "%autochangelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("upstream changelog"), fileperms.PublicFile)) + + stub := &renderAutospecStub{err: testCase.releaseError} + err = prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + lastRender{spec: committed, tree: tree}) + require.ErrorContains(t, err, testCase.want) + + if testCase.newVersion && !testCase.missingFile && !testCase.missingTree { + assert.Equal(t, []string{"release"}, stub.calls) + } else { + assert.Empty(t, stub.calls) + } + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Equal(t, original, string(content)) + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + + wantChangelog := "committed changelog" + if testCase.missingFile || testCase.missingTree { + wantChangelog = "upstream changelog" + } + + assert.Equal(t, wantChangelog, string(content)) + }) + } +} + +func TestLocklessRenderRelease_StaticHistoryWithoutAutoreleaseMarker(t *testing.T) { + for _, testCase := range []struct{ header, release string }{ + {"## RPMAUTOSPEC: autochangelog\n", "%autorelease"}, + {"## RPMAUTOSPEC: autochangelog\nautorelease\n", "%autorelease"}, + {" ## RPMAUTOSPEC: autorelease\n", "%autorelease"}, + {"## RPMAUTOSPEC: autorelease\n", "1%{?dist}"}, + } { + env := testutils.NewTestEnv(t) + stub := &renderAutospecStub{} + history := lastRender{spec: testCase.header + renderTestSpec("1", "9", "old\n")} + update, err := locklessRenderRelease(env.Env, stub, "/working", "curl.spec", testCase.release, history, false) + require.NoError(t, err) + got, err := update(testCase.release) + require.NoError(t, err) + assert.Equal(t, testCase.release, got) + assert.Empty(t, stub.calls) + } +} + +func TestPrepareLocklessSpec_CommittedAutoreleaseFailure(t *testing.T) { + for _, testCase := range []struct { + name, output, want string + err error + }{ + {name: "query failure", err: errors.New("query failed"), want: "expanding committed Release"}, + {name: "unresolved release", output: "%{unknown}", want: "using committed Release"}, + {name: "zero base", output: "0.fc43", want: "using committed Release"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + original := renderTestSpec("2", "%autorelease", "%autochangelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("original changelog"), fileperms.PublicFile)) + + stub := &renderAutospecStub{expand: func(_, _ string) (string, error) { + return testCase.output, testCase.err + }} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, + testCommittedAutoreleaseHistory(t)) + require.ErrorContains(t, err, testCase.want) + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Equal(t, original, string(content)) + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, carriedAutoreleaseChangelog, string(content)) + }) + } +} + +func TestWithAutoreleaseBase(t *testing.T) { + for _, testCase := range []struct{ release, want, wantDefault string }{ + {"%autorelease", "%autorelease -b 42", "%autorelease"}, + {"%autorelease -b 7 -e test", "%autorelease -b 42 -e test", "%autorelease -e test"}, + {"%autorelease -b7", "%autorelease -b 42", "%autorelease"}, + {"%autorelease -b=7 -e test", "%autorelease -b 42 -e test", "%autorelease -e test"}, + {"%{autorelease}", "%{autorelease -b 42}", "%{autorelease}"}, + {"%{autorelease -b 7}", "%{autorelease -b 42}", "%{autorelease}"}, + {"%autorelease -e %{suffix}", "%autorelease -e %{suffix} -b 42", "%autorelease -e %{suffix}"}, + {"%{autorelease}%{?dist}", "%{autorelease -b 42}%{?dist}", "%{autorelease}%{?dist}"}, + {"%{autorelease -e %{suffix}}", "%{autorelease -e %{suffix} -b 42}", "%{autorelease -e %{suffix}}"}, + } { + got, err := withAutoreleaseBase(testCase.release, "Calculated release number: 42\n", true) + require.NoError(t, err) + assert.Equal(t, testCase.want, got) + + got, err = withAutoreleaseBase(testCase.release, "Calculated release number: 1\n", true) + require.NoError(t, err) + assert.Equal(t, testCase.wantDefault, got) + } + + for _, output := range []string{"", "release: invalid", "0", "7.fc43"} { + _, err := withAutoreleaseBase("%autorelease", output, true) + require.Error(t, err) + } +} + +func TestPrepareComponentSources_LockedSkipNewLifecycle(t *testing.T) { + for _, testCase := range []struct { + name string + source projectconfig.SpecSourceType + lockless bool + }{ + {name: "locked local", source: projectconfig.SpecSourceTypeLocal}, + {name: "locked upstream", source: projectconfig.SpecSourceTypeUpstream}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: testCase.source}, + }) + + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).Return(nil) + + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { newSourcePreparer = original }) + + _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/rendered", + spec.EditorLegacy, false, testCase.lockless) + require.ErrorContains(t, err, "failed to apply overlays") + }) + } +} + +func TestPrepareComponentSources_LocalLocklessKeepsSpecAndSetsCommitState(t *testing.T) { + env := testutils.NewTestEnv(t) + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + }) + + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("local project")}}}) + originalRepo := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { return repo, renderTestProjectRoot, nil } + + t.Cleanup(func() { openRenderRepo = originalRepo }) + + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", "7%{?dist}", "* Keep local changelog\n")), fileperms.PublicFile) + }) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { newSourcePreparer = originalPreparer }) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected upstream history or release query: %v", cmd.Args) + + return "", nil + } + prep, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/SPECS/c/curl", + spec.EditorLegacy, true, true) + require.NoError(t, err) + require.NotNil(t, prep.update) + assert.True(t, prep.update.local) + assert.Equal(t, renderTestProjectRoot, prep.update.repoRoot) + assert.Equal(t, "SPECS/c/curl", prep.update.path) + + content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: 7%{?dist}") + assert.Contains(t, string(content), "* Keep local changelog\n") +} + +func TestFinishLocalLocklessSpec_DoesNotBump(t *testing.T) { + for _, testCase := range []struct { + name, release string + changed, rebuild bool + wantError error + wantMarker bool + }{ + {name: "changed static", release: "7%{?dist}", changed: true}, + {name: "unchanged static", release: "7%{?dist}", wantError: errLocalManualRebuild}, + {name: "static rebuild refused", release: "7%{?dist}", rebuild: true, wantError: errLocalManualRebuild}, + {name: "unchanged autorelease", release: "%autorelease -b 7", wantError: errNoRenderChanges}, + {name: "autorelease rebuild", release: "%autorelease -b 7", rebuild: true, wantMarker: true}, + {name: "changed autorelease", release: "%autorelease -b 7", changed: true, rebuild: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("date") + + path := "/working/curl.spec" + content := renderTestSpec("1", testCase.release, "* Keep local changelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(content), fileperms.PublicFile)) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{"date", "-Is"}, cmd.Args) + + return "2026-11-06T16:00:00+00:00\n", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("local specs must never invoke bumpspec or a changelog editor: %v", cmd.Args) + + return nil + } + + changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{local: true, noGitCommit: true}, + testCase.changed, testCase.rebuild) + if testCase.wantError != nil { + require.ErrorIs(t, err, testCase.wantError) + } else { + require.NoError(t, err) + assert.Equal(t, testCase.wantMarker, changed) + } + + got, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Equal(t, content, string(got)) + + exists, err := fileutils.Exists(env.TestFS, "/working/.no-change-rebuild") + require.NoError(t, err) + assert.Equal(t, testCase.wantMarker, exists) + }) + } +} + +func assertCurlRenderCommit(t *testing.T, args []string, message string, provided bool) { + t.Helper() + + wantArgs := []string{"git", "-C", "/project", "commit", "--only", "-m", message} + if provided { + wantArgs = append(wantArgs, "-m", "abc123 Fix upstream") + } + + wantArgs = append(wantArgs, "--", "SPECS/c/curl") + assert.Equal(t, wantArgs, args) +} + +func finishPreparedCurlRender(t *testing.T, env *testutils.TestEnv, prep *preparedComponent) (bool, error) { + t.Helper() + stubIncreasingRenderNEVR(t, env) + + return finishComponentRender(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) +} + +func TestPrepareComponentSources_ChangelogAndReleaseBeforeOverlays(t *testing.T) { + const releaseTag = "Release: " + + for _, testCase := range []struct { + name, comment, wantComment string + upstreamVersion, overlayVersion, wantRelease string + explicit, noGitCommit, skipReleaseOverlay bool + }{ + {name: "edited comment", wantComment: "Edited changelog text"}, + { + name: "custom comment", comment: "Fix curl's patches; keep $COMPONENT literal", + wantComment: "Fix curl's patches; keep $COMPONENT literal", explicit: true, + }, + {name: "empty explicit comment", explicit: true}, + { + name: "no Git commit", comment: "Prepare curl for review", + wantComment: "Prepare curl for review", noGitCommit: true, + }, + { + name: "version overlay cannot enable carryover", comment: "update curl", wantComment: "update curl", + upstreamVersion: "2", overlayVersion: "1", wantRelease: "1", skipReleaseOverlay: true, + }, + { + name: "version overlay cannot disable carryover", comment: "update curl", wantComment: "update curl", + upstreamVersion: "1", overlayVersion: "2", wantRelease: "9", skipReleaseOverlay: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + ctrl := gomock.NewController(t) + + upstreamVersion, overlayVersion, wantRelease := "2", "1", "2" + if testCase.skipReleaseOverlay { + upstreamVersion = testCase.upstreamVersion + overlayVersion = testCase.overlayVersion + wantRelease = testCase.wantRelease + } + + releaseNumber, err := strconv.Atoi(wantRelease) + require.NoError(t, err) + + wantBumped := strconv.Itoa(releaseNumber + 1) + + config := &projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + Build: projectconfig.ComponentBuildConfig{With: []string{"feature"}}, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Version", Value: overlayVersion}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid summary"}, + { + Type: projectconfig.ComponentOverlaySearchAndReplaceInSpec, SectionName: "%changelog", + Regex: "Downstream entry", Replacement: "Overlaid entry", + }, + }, + SourceFiles: []projectconfig.SourceFileReference{{ + Filename: "extra-source.tar.gz", Hash: "abc123", HashType: fileutils.HashTypeSHA512, + Origin: projectconfig.Origin{Type: projectconfig.OriginTypeURI, Uri: "https://example.com/extra-source.tar.gz"}, + }}, + } + if testCase.skipReleaseOverlay { + config.Overlays = config.Overlays[1:] + } + + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(config) + + const ( + oldChangelog = "* Downstream entry\n" + rootConfig = `[project] +default-distro = { name = "test-distro", version = "1.0" } +[distros.test-distro.versions."1.0"] +release-ver = "3" +dist-git-branch = "upstream" +[components.curl.spec] +type = "upstream" +upstream-commit = "aaaaaaa" +` + ) + + repo, hashes := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "azldev.toml": []byte(rootConfig), + "SPECS/c/curl/curl.spec": []byte(renderTestSpec("1", "9", oldChangelog)), + }}}) + originalOpen := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { + return repo, "/project", nil + } + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + upstream := strings.Replace(renderTestSpec(upstreamVersion, "1", "* Upstream entry\n"), + "Release: ", releaseTag, 1) + + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(upstream), fileperms.PublicFile) + }) + + originalPreparer := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { + openRenderRepo = originalOpen + newSourcePreparer = originalPreparer + }) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + bumped := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/project", "--literal-pathspecs", "log", "-1", "--format=%H", + "--end-of-options", hashes[0], "--", "SPECS/c/curl/", + }, cmd.Args) + + return hashes[0], nil + } + env.CmdFactory.RegisterCommandInSearchPath("test-editor") + setRenderEditor(t, env, "test-editor") + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if cmd.Args[0] == "test-editor" { + return fileutils.WriteFile(env.TestFS, cmd.Args[len(cmd.Args)-1], + []byte(testCase.wantComment), fileperms.PrivateFile) + } + + bumped = true + + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + assert.Equal(t, "-c", cmd.Args[1]) + assert.Equal(t, testCase.wantComment, cmd.Args[2]) + path := cmd.Args[len(cmd.Args)-1] + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "Summary: Overlaid summary\n") + assert.Contains(t, string(content), releaseTag+wantRelease+"\n") + assert.Contains(t, string(content), "Version: "+overlayVersion+"\n") + assert.Contains(t, string(content), "%changelog\n* Overlaid entry\n") + assert.Contains(t, string(content), "%define _with_feature 1\n") + assert.NotContains(t, string(content), "%{load:") + assert.NotContains(t, string(content), "Source9999") + + exists, err := fileutils.Exists(env.TestFS, "/staging/curl/curl.azl.macros") + require.NoError(t, err) + assert.False(t, exists) + + manifest, err := fileutils.ReadFile(env.TestFS, "/staging/curl/sources") + require.NoError(t, err) + assert.Contains(t, string(manifest), "SHA512 (extra-source.tar.gz) = abc123") + + return fileutils.WriteFile(env.TestFS, path, + []byte(strings.Replace(string(content), releaseTag+wantRelease+"\n", + releaseTag+wantBumped+"\n", 1)), fileperms.PublicFile) + } + results := make([]*RenderResult, 1) + prepared := parallelPrepare(env.Env, nil, []components.Component{comp}, "/staging", &RenderOptions{ + OutputDir: "/project/SPECS", ChangelogComment: testCase.comment, + changelogCommentExplicit: testCase.explicit, NoGitCommit: testCase.noGitCommit, + }, results) + require.Len(t, prepared, 1, "preparation results: %+v", results[0]) + prep := prepared[0] + prep.update.changes = "abc123 Fix upstream" + + assert.False(t, bumped) + + prep.compOutputDir = "/project/SPECS/c/curl" + changed, err := finishPreparedCurlRender(t, env, prep) + require.NoError(t, err) + assert.True(t, changed) + assert.True(t, bumped) + + content, err := fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), releaseTag+wantBumped+"\n") + + var gitCommands []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + gitCommands = append(gitCommands, cmd.Args[3]) + + assert.False(t, testCase.noGitCommit, "no Git commands should run for --no-git-commit") + + if cmd.Args[3] == renderTestDiffCommand { + if slices.Contains(cmd.Args, "--name-status") { + return "", nil + } + + return renderTestSpecDiff, nil + } + + if cmd.Args[3] == "commit" { + assertCurlRenderCommit(t, cmd.Args, testCase.wantComment, + testCase.explicit || testCase.comment != "") + } + + return "", nil + } + results[0] = &RenderResult{Status: renderStatusOK, lockless: true} + commitLocklessRenders(env.Env, prepared, results, false) + + if testCase.noGitCommit { + assert.Empty(t, gitCommands) + } else { + assert.Equal(t, []string{ + "add", renderTestDiffCommand, "add", renderTestDiffCommand, "commit", + }, gitCommands) + } + + assert.Equal(t, renderStatusOK, results[0].Status) + + content, err = fileutils.ReadFile(env.TestFS, "/project/SPECS/c/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), releaseTag+wantBumped+"\n") + }) + } +} + +func TestPrepareComponentSources_LocklessSkipsSyntheticHistory(t *testing.T) { + for _, release := range []string{"7%{?dist}", "%autorelease -b 9"} { + t.Run(release, func(t *testing.T) { + env := testutils.NewTestEnv(t) + repo, _ := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{"README": []byte("local")}}}) + originalRepo := openRenderRepo + openRenderRepo = func(_ *azldev.Env) (*gogit.Repository, string, error) { return repo, renderTestProjectRoot, nil } + + t.Cleanup(func() { openRenderRepo = originalRepo }) + ctrl := gomock.NewController(t) + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + }) + + manager := sourceproviders_test.NewMockSourceManager(ctrl) + + const changelog = "* Existing entry\n- Keep unchanged\n" + + manager.EXPECT().FetchComponent(gomock.Any(), comp, "/staging/curl", gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, opts ...sourceproviders.FetchComponentOption) error { + var resolved sourceproviders.FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + assert.True(t, resolved.PreserveGitDir) + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(dir, ".git/HEAD"), + []byte("upstream history"), fileperms.PublicFile)) + + return fileutils.WriteFile(env.TestFS, filepath.Join(dir, "curl.spec"), + []byte(renderTestSpec("1", release, changelog)), fileperms.PublicFile) + }) + + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, events opctx.EventListener, dry opctx.DryRunnable, + options ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return sources.NewPreparer(manager, fs, events, dry, options...) + } + + t.Cleanup(func() { newSourcePreparer = original }) + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected synthetic-history command: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("unexpected synthetic-history query: %v", cmd.Args) + + return "", nil + } + + _, err := prepareComponentSources(env.Env, nil, comp, "/staging", "/project/rendered", + spec.EditorLegacy, false, true) + require.NoError(t, err) + content, err := fileutils.ReadFile(env.TestFS, "/staging/curl/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "Release: "+release+"\n") + assert.Contains(t, string(content), "%changelog\n"+changelog) + content, err = fileutils.ReadFile(env.TestFS, "/staging/curl/.git/HEAD") + require.NoError(t, err) + assert.Equal(t, "upstream history", string(content)) + }) + } +} + +func TestPrepareLocklessSpec_DetectsVersionBeforeCarryover(t *testing.T) { + env := testutils.NewTestEnv(t) + original := renderTestSpec("2", "%autorelease", "%autochangelog\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/changelog", + []byte("original changelog"), fileperms.PublicFile)) + + history := testCommittedAutoreleaseHistory(t) + history.spec = strings.Replace(history.spec, "Version: 1\n", "", 1) + stub := &renderAutospecStub{expand: func(_, _ string) (string, error) { + return "9", nil + }} + + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, history) + require.ErrorContains(t, err, "reading committed Version") + assert.Empty(t, stub.calls) + + content, err := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, err) + assert.Equal(t, original, string(content)) + content, err = fileutils.ReadFile(env.TestFS, "/working/changelog") + require.NoError(t, err) + assert.Equal(t, "original changelog", string(content)) +} + +func TestPrepareLocklessSpec_StopsOnAutospecFailure(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec("1", "%autorelease", "%autochangelog\n")), fileperms.PublicFile)) + + stub := &renderAutospecStub{err: errors.New("autospec failed")} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", spec.EditorLegacy, lastRender{}) + require.ErrorContains(t, err, "autospec failed") + assert.Equal(t, []string{"changelog"}, stub.calls) +} + +func TestFinishLocklessSpec_TimestampFailures(t *testing.T) { + for _, testCase := range []struct { + name, want string + available bool + dateFails bool + readOnly bool + }{ + {name: "date missing", want: "requires 'date'"}, + {name: "date fails", available: true, dateFails: true, want: "getting rebuild timestamp"}, + {name: "write fails", available: true, readOnly: true, want: "writing rebuild timestamp"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/working/curl.spec" + + original := renderTestSpec("1", "%autorelease -b 9", "") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(original), fileperms.PublicFile)) + + if testCase.readOnly { + env.TestFS = afero.NewReadOnlyFs(env.TestFS) + } + + if testCase.available { + env.CmdFactory.RegisterCommandInSearchPath("date") + } + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not bump without a timestamp: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{"date", "-Is"}, cmd.Args) + + if testCase.dateFails { + return "", errors.New("date failed") + } + + return "2026-11-04T22:53:27+00:00\n", nil + } + + changed, err := finishLocklessSpec(env.Env, path, &locklessRenderUpdate{}, false, true) + require.ErrorContains(t, err, testCase.want) + assert.False(t, changed) + + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Equal(t, original, string(content)) + + exists, err := fileutils.Exists(env.TestFS, filepath.Join("/working", noChangeRebuildMarker)) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func TestBumpLocklessSpec_Failures(t *testing.T) { + for _, available := range []bool{false, true} { + t.Run(strconv.FormatBool(available), func(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(renderTestSpec("1", "1", "")), fileperms.PublicFile)) + + if available { + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + } + + called := false + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + called = true + + return errors.New("invalid spec") + } + _, err := bumpLocklessSpec(env.Env, "/working/curl.spec", &locklessRenderUpdate{subject: "updated curl"}) + + assert.Equal(t, available, called) + + if available { + require.ErrorContains(t, err, "invalid spec") + } else { + require.ErrorContains(t, err, "install 'rpmdevtools'") + } + }) + } +} + +func TestBumpLocklessSpec_IgnoresChangelogReleaseLines(t *testing.T) { + const path = "/working/curl.spec" + + changelog := "* Previous\n- Example:\nRelease: %%{release_string}.2\nrelease : 3\n" + before := renderTestSpec("1", "1", changelog) + + for _, fail := range []bool{false, true} { + t.Run(strconv.FormatBool(fail), func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(before), fileperms.PublicFile)) + + // Mimic rpmdev-bumpspec, which bumps every line starting with 'Release:'. + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + current, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + + bumped := regexp.MustCompile(`(?mi)^release\s*:.*$`).ReplaceAll(current, []byte("Release: BUMPED")) + require.NoError(t, fileutils.WriteFile(env.TestFS, path, bumped, fileperms.PublicFile)) + + if fail { + return errors.New("bumpspec failed") + } + + return nil + } + + changed, err := bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "Render message"}) + + result, readErr := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, readErr) + + if fail { + require.ErrorContains(t, err, "bumpspec failed") + assert.False(t, changed) + assert.Equal(t, before, string(result)) + + return + } + + require.NoError(t, err) + assert.True(t, changed) + assert.Equal(t, renderTestSpec("1", "BUMPED", changelog), string(result)) + }) + } +} + +func TestBumpLocklessSpec_ProcessLocalRPMContext(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(strconv.FormatBool(fail), func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + const path = "/working/spec dir's/curl.spec" + + before := renderTestSpec("1", "1", "") + after := renderTestSpec("1", "2", "- Render message\n") + + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(before), fileperms.PublicFile)) + + var wrapperDir string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", "Render message", path}, cmd.Args) + assert.Equal(t, filepath.Dir(path), cmd.Dir) + wrapperDir = filepath.SplitList(renderCommandEnvValue(cmd.Env, "PATH"))[0] + wrapper, err := fileutils.ReadFile(env.TestFS, filepath.Join(wrapperDir, "rpm")) + require.NoError(t, err) + assert.Contains(t, string(wrapper), "'_sourcedir /working/spec dir'\"'\"'s'") + assert.Contains(t, string(wrapper), "PATH=\"$RPMDEV_BUMPSPEC_ORIGINAL_PATH\"") + assert.Equal(t, env.OSEnv().Getenv("PATH"), + renderCommandEnvValue(cmd.Env, "RPMDEV_BUMPSPEC_ORIGINAL_PATH")) + + inherited := (&exec.Cmd{}).Environ() + for _, key := range []string{"HOME", "LC_ALL", "TZ"} { + assert.Equal(t, renderCommandEnvValue(inherited, key), renderCommandEnvValue(cmd.Env, key)) + } + + if fail { + return errors.New("bumpspec failed") + } + + return fileutils.WriteFile(env.TestFS, path, []byte(after), fileperms.PublicFile) + } + + changed, err := bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "Render message"}) + if fail { + require.ErrorContains(t, err, "bumpspec failed") + assert.False(t, changed) + } else { + require.NoError(t, err) + assert.True(t, changed) + } + + require.NotEmpty(t, wrapperDir) + exists, err := fileutils.Exists(env.TestFS, wrapperDir) + require.NoError(t, err) + assert.False(t, exists) + }) + } +} + +func renderCommandEnvValue(values []string, key string) string { + for _, value := range slices.Backward(values) { + if text, ok := strings.CutPrefix(value, key+"="); ok { + return text + } + } + + return "" +} + +func TestPrepareRenderBumpspecEnvironment_IsolatedInvocations(t *testing.T) { + env := testutils.NewTestEnv(t) + first := &exec.Cmd{Env: []string{"HOME=/user", "LC_ALL=custom", "TZ=custom", "KEEP=value"}} + second := &exec.Cmd{} + firstDir, err := prepareRenderBumpspecEnvironment(env.Env, first, "/first/curl.spec") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, env.TestFS.RemoveAll(firstDir)) }) + + secondDir, err := prepareRenderBumpspecEnvironment(env.Env, second, "/second/curl.spec") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, env.TestFS.RemoveAll(secondDir)) }) + + assert.NotEqual(t, firstDir, secondDir) + + for _, item := range []string{"HOME=/user", "LC_ALL=custom", "TZ=custom", "KEEP=value"} { + assert.Contains(t, first.Env, item) + } + + require.NoError(t, env.TestFS.RemoveAll(firstDir)) + + wrapper, err := fileutils.ReadFile(env.TestFS, filepath.Join(secondDir, "rpm")) + require.NoError(t, err) + assert.Contains(t, string(wrapper), "'_sourcedir /second'") + assert.NotContains(t, string(wrapper), "'_sourcedir /first'") +} + +type renderBumpspecFaultFS struct { + afero.Fs + failure string + dir string +} + +func (fs *renderBumpspecFaultFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + if filepath.Base(name) == "rpm" { + fs.dir = filepath.Dir(name) + if fs.failure == "write" { + return nil, errors.New("injected wrapper write failure") + } + } + + return fs.Fs.OpenFile(name, flag, perm) +} + +func (fs *renderBumpspecFaultFS) RemoveAll(path string) error { + if path == fs.dir && fs.failure == renderTestWrapperCleanupFailure { + return errors.New("injected wrapper cleanup failure") + } + + return fs.Fs.RemoveAll(path) +} + +func TestBumpLocklessSpec_WrapperFilesystemFailures(t *testing.T) { + for _, failure := range []string{"write", renderTestWrapperCleanupFailure} { + t.Run(failure, func(t *testing.T) { + env := testutils.NewTestEnv(t) + faultFS := &renderBumpspecFaultFS{Fs: env.TestFS, failure: failure} + env.TestFS = faultFS + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + path := "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec("1", "1", "")), fileperms.PublicFile)) + + called := false + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + called = true + + return fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec("1", "2", "")), fileperms.PublicFile) + } + + _, err := bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "Render message"}) + require.ErrorContains(t, err, "injected wrapper "+failure+" failure") + assert.Equal(t, failure == renderTestWrapperCleanupFailure, called) + require.NotEmpty(t, faultFS.dir) + exists, err := fileutils.Exists(env.TestFS, faultFS.dir) + require.NoError(t, err) + assert.Equal(t, failure == renderTestWrapperCleanupFailure, exists) + }) + } +} + +func TestCommitLocklessRenders_CheckOnlyAndFailure(t *testing.T) { + for _, checkOnly := range []bool{false, true} { + t.Run(strconv.FormatBool(checkOnly), func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/project/SPECS/c/curl/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte("rendered output"), fileperms.PublicFile)) + + called := false + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + called = true + + return "", errors.New("git identity is missing") + } + prepared := []*preparedComponent{{ + update: &locklessRenderUpdate{repoRoot: "/project", path: "SPECS/c/curl"}, + }} + results := []*RenderResult{{Status: renderStatusOK}} + commitLocklessRenders(env.Env, prepared, results, checkOnly) + assert.Equal(t, !checkOnly, called) + + if checkOnly { + assert.Equal(t, renderStatusOK, results[0].Status) + } else { + assert.Equal(t, renderStatusError, results[0].Status) + assert.Contains(t, results[0].Error, "git identity is missing") + } + + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Equal(t, "rendered output", string(content)) + }) + } +} + +func TestRestoreRenderChangelog_ExactBytesAndOtherSections(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + previous := []byte(renderTestSpec("1", "9", "* Old entry\n- Whitespace \n\n")) + upstream := []byte(renderTestSpec("2", "1", "* New entry\n- Discard\n") + "%files\n/file\n") + got, err := restoreRenderChangelog(upstream, previous, editor) + require.NoError(t, err) + assert.Contains(t, string(got), "%changelog\n* Old entry\n- Whitespace \n\n%files\n/file\n") + assert.NotContains(t, string(got), "Discard") + }) + } +} + +func TestRestoreRenderChangelog_MissingSection(t *testing.T) { + const oldChangelog = "%changelog\n* Old entry\n- Keep\n" + for _, testCase := range []struct{ upstream, previous, want string }{ + {"Name: curl", oldChangelog, "Name: curl\n" + oldChangelog}, + {"Name: curl\n", oldChangelog, "Name: curl\n" + oldChangelog}, + {"Name: curl", "Name: old\n", "Name: curl"}, + {"Name: curl\n%changelog\nDiscard\n", "Name: old\n", "Name: curl\n"}, + } { + got, err := restoreRenderChangelog([]byte(testCase.upstream), []byte(testCase.previous), spec.EditorLegacy) + require.NoError(t, err) + assert.Equal(t, testCase.want, string(got)) + } +} + +func TestCommitLocklessRenders_SkipsFailedComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + + var commands [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args) + + if cmd.Args[3] == renderTestDiffCommand { + if slices.Contains(cmd.Args, "--name-status") { + return "M\x00config.toml\x00", nil + } + + return "SPECS/v/vim/vim.spec\n", nil + } + + return "", nil + } + prepared := []*preparedComponent{ + {index: 0}, + {index: 1, update: &locklessRenderUpdate{repoRoot: "/project", path: "SPECS/c/curl"}}, + {index: 2, update: &locklessRenderUpdate{ + repoRoot: "/project", path: "SPECS/v/vim", subject: "updated vim", + }}, + } + results := []*RenderResult{ + {Status: renderStatusOK}, + {Status: renderStatusError, Error: "render failed"}, + {Status: renderStatusOK}, + } + commitLocklessRenders(env.Env, prepared, results, false) + assert.Equal(t, "render failed", results[1].Error) + assert.Equal(t, renderStatusOK, results[2].Status) + require.Len(t, commands, 5) + assert.Equal(t, []string{ + "git", "-C", "/project", "add", "--all", "--", ".", ":(top,exclude,literal)SPECS", + }, commands[0]) + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", "updated vim", + "--", "SPECS/v/vim", ":(top,literal)config.toml", + }, commands[4]) +} + +func TestCommitLocklessRenders_InputsOnlyInFirstCommit(t *testing.T) { + for _, testCase := range []struct { + name string + empty, fail bool + wantInputQueries int + }{ + {name: "successful first commit", wantInputQueries: 1}, + {name: "empty first attempt", empty: true, wantInputQueries: 2}, + {name: "failed first commit", fail: true, wantInputQueries: 2}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + inputQueries := 0 + + var commits [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + switch cmd.Args[3] { + case "add": + if cmd.Args[len(cmd.Args)-1] == ":(top,exclude,literal)nested/rendered" { + assert.Equal(t, ".", cmd.Args[len(cmd.Args)-2]) + } + case renderTestDiffCommand: + if slices.Contains(cmd.Args, "--name-status") { + inputQueries++ + + assert.Contains(t, cmd.Args, ":(top,exclude,literal)nested/rendered") + assert.Contains(t, cmd.Args, "--no-renames") + assert.Contains(t, cmd.Args, "-z") + + if testCase.empty && inputQueries == 1 { + return "", nil + } + + return "M\x00config.toml\x00A\x00 scratch[1].txt\x00D\x00removed.txt\x00", nil + } + + if testCase.empty && slices.Contains(cmd.Args, "nested/rendered/c/curl") { + return "", nil + } + + return "changed\n", nil + case "commit": + commits = append(commits, cmd.Args) + if testCase.fail && slices.Contains(cmd.Args, "update curl") { + return "", errors.New("first commit failed") + } + default: + t.Fatalf("unexpected Git command: %v", cmd.Args) + } + + return "", nil + } + prepared := []*preparedComponent{ + {index: 0, update: &locklessRenderUpdate{ + repoRoot: "/project", path: "nested/rendered/c/curl", subject: "update curl", + }}, + {index: 1, update: &locklessRenderUpdate{ + repoRoot: "/project", path: "nested/rendered/v/vim", subject: "update vim", + }}, + } + results := []*RenderResult{{Status: renderStatusOK}, {Status: renderStatusOK}} + + commitLocklessRenders(env.Env, prepared, results, false) + + assert.Equal(t, testCase.wantInputQueries, inputQueries) + require.NotEmpty(t, commits) + firstCommit := commits[0] + assert.Contains(t, firstCommit, ":(top,literal)config.toml") + assert.Contains(t, firstCommit, ":(top,literal) scratch[1].txt") + assert.Contains(t, firstCommit, ":(top,literal)removed.txt") + + if !testCase.empty { + assert.Contains(t, firstCommit, "nested/rendered/c/curl") + assert.NotContains(t, firstCommit, "nested/rendered/v/vim") + } + + lastCommit := commits[len(commits)-1] + assert.Contains(t, lastCommit, "nested/rendered/v/vim") + assert.NotContains(t, lastCommit, "nested/rendered/c/curl") + assert.Equal(t, renderStatusOK, results[1].Status) + + if testCase.fail || testCase.empty { + assert.Contains(t, lastCommit, ":(top,literal)config.toml") + } else { + assert.NotContains(t, lastCommit, ":(top,literal)config.toml") + } + }) + } +} + +func TestCommitLocklessRender_PathOnlyAndNoEmptyCommits(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(strconv.FormatBool(changed), func(t *testing.T) { + env := testutils.NewTestEnv(t) + + var commands [][]string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + commands = append(commands, cmd.Args) + if cmd.Args[3] == renderTestDiffCommand && changed { + return renderTestSpecDiff, nil + } + + return "", nil + } + update := &locklessRenderUpdate{ + repoRoot: "/project", path: "SPECS/c/curl", subject: "updated curl", changes: "abc123 Fix", + } + committed, err := commitLocklessRender(env.Env, update, false) + require.NoError(t, err) + assert.Equal(t, changed, committed) + assert.Equal(t, []string{"git", "-C", "/project", "add", "--all", "--", update.path}, commands[0]) + + if changed { + require.Len(t, commands, 3) + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", "updated curl", + "--", "SPECS/c/curl", + }, commands[2]) + } else { + assert.Len(t, commands, 2) + } + }) + } +} + +func conditionalReleaseTestSpec(withRelease, withoutRelease string) string { + return fmt.Sprintf("Name: curl\nVersion: 1\n%%if %%{with release}\nRelease: %s\n%%else\n"+ + "Release: %s\n%%endif\nSummary: Test\nLicense: MIT\n\n%%description\nTest\n\n%%changelog\n%%autochangelog\n", + withRelease, withoutRelease) +} + +func TestPrepareLocklessSpec_ConditionalReleaseTags(t *testing.T) { + const snapshot = "%autorelease -s %{gitdate}git%{shortcommit}" + + unprocessed := conditionalReleaseTestSpec("%autorelease -b 5", snapshot+" -b 5") + unprocessedRepo, unprocessedCommits := testRepoWithCommits(t, []testRepoCommit{{files: map[string][]byte{ + "curl.spec": []byte(unprocessed), "changelog": []byte("committed changelog"), + }}}) + unprocessedTree, err := resolveTree(unprocessedRepo, unprocessedCommits[0]) + require.NoError(t, err) + + for _, testCase := range []struct { + name, upstreamFirst, upstreamSecond string + history lastRender + want []string + wantErr error + }{ + { + name: "initial autorelease", upstreamFirst: "%autorelease", upstreamSecond: snapshot, + want: []string{"%autorelease -b 42", snapshot + " -b 42"}, + }, + { + name: "unprocessed autorelease history", upstreamFirst: "%autorelease", upstreamSecond: snapshot, + history: lastRender{spec: unprocessed, tree: unprocessedTree}, + want: []string{"%autorelease -b 5", snapshot + " -b 5"}, + }, + { + name: "rendered autorelease history", upstreamFirst: "%autorelease", upstreamSecond: snapshot, + history: testCommittedAutoreleaseHistory(t), + want: []string{"%autorelease -b 9", snapshot + " -b 9"}, + }, + { + name: "static release history", upstreamFirst: "1%{?dist}", upstreamSecond: "1.%{snap}%{?dist}", + history: lastRender{spec: conditionalReleaseTestSpec("5%{?dist}", "5.%{snap}%{?dist}")}, + want: []string{"5%{?dist}", "5.%{snap}%{?dist}"}, + }, + { + name: "static release count mismatch", upstreamFirst: "1%{?dist}", upstreamSecond: "1.%{snap}%{?dist}", + history: lastRender{spec: renderTestSpec("1", "5%{?dist}", "")}, + wantErr: ErrReleaseTagCountMismatch, + }, + { + name: "mixed upstream release", upstreamFirst: "%autorelease", upstreamSecond: "1%{?dist}", + wantErr: ErrMixedReleaseTags, + }, + { + name: "differing committed autorelease", upstreamFirst: "1%{?dist}", upstreamSecond: "1.%{snap}%{?dist}", + history: lastRender{spec: conditionalReleaseTestSpec("%autorelease", snapshot)}, + wantErr: ErrDifferingAutoreleaseTags, + }, + } { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(testCase.name+"/"+string(editor), func(t *testing.T) { + env := testutils.NewTestEnv(t) + original := conditionalReleaseTestSpec(testCase.upstreamFirst, testCase.upstreamSecond) + require.NoError(t, fileutils.WriteFile(env.TestFS, "/working/curl.spec", + []byte(original), fileperms.PublicFile)) + + stub := &renderAutospecStub{expand: func(_, _ string) (string, error) { return "9.fc43", nil }} + err := prepareLocklessSpec(env.Env, stub, "/working", "curl", editor, testCase.history) + + content, readErr := fileutils.ReadFile(env.TestFS, "/working/curl.spec") + require.NoError(t, readErr) + + if testCase.wantErr != nil { + require.ErrorIs(t, err, testCase.wantErr) + assert.Equal(t, original, string(content)) + + return + } + + require.NoError(t, err) + + prepared, err := spec.OpenSpec(strings.NewReader(string(content)), spec.WithEditor(editor)) + require.NoError(t, err) + releases, err := prepared.GetTags("", "Release") + require.NoError(t, err) + assert.Equal(t, testCase.want, releases) + }) + } + } +} diff --git a/internal/app/azldev/cmds/component/render_message.go b/internal/app/azldev/cmds/component/render_message.go new file mode 100644 index 000000000..f52297c43 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_message.go @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/kballard/go-shellquote" + "github.com/mattn/go-isatty" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" +) + +//nolint:gochecknoglobals // Replaced by isolated render tests without changing real standard streams. +var renderIsTerminal = isatty.IsTerminal + +const renderChangelogTemplate = "\n\n" + + "# Please enter a changelog entry message. The text will be used\n" + + "# unmodified for the changelog entry (other than the first line\n" + + "# automatically getting a prefix of '- '. This will also be used for\n" + + "# the git commit message.\n" + +const renderNoGitChangelogTemplate = "\n\n" + + "# Please enter a changelog entry message. The text will be used\n" + + "# unmodified for the changelog entry (other than the first line\n" + + "# automatically getting a prefix of '- '.\n" + +const renderAutoreleaseTemplate = "\n\n" + + "# Please enter a changelog entry message. The first line will be used\n" + + "# for the changelog entry and git commit summary, and the following\n" + + "# lines will be used as the git commit message.\n" + + "#\n" + + "# To include more lines in the changelog entry, use this reference:\n" + + "# https://fedora-infra.github.io/rpmautospec-docs/autochangelog.html" + + "#changelog-entries-generated-from-commit-messages\n" + +const renderLocalCommitTemplate = "\n\n" + + "# Please provide a git commit message. This is not used for the\n" + + "# changelog, this is used only for the git commit.\n" + +func renderFinishConcurrency(env *azldev.Env, prepared []*preparedComponent, checkOnly bool) int { + if !checkOnly && !env.DryRun() { + for _, prep := range prepared { + if prep.update != nil && (!prep.update.local || !prep.update.noGitCommit) && + !prep.update.subjectProvided && prep.update.subject == "" { + return 1 + } + } + } + + return env.IOBoundConcurrency() +} + +func renderEditor(env *azldev.Env) ([]string, error) { + if editor := env.OSEnv().Getenv("EDITOR"); editor != "" { + args, err := shellquote.Split(editor) + if err != nil { + return nil, fmt.Errorf("parsing 'EDITOR':\n%w", err) + } + + if len(args) == 0 || !env.CommandInSearchPath(args[0]) { + return nil, fmt.Errorf("editor configured by 'EDITOR' is unavailable: %#q", editor) + } + + return args, nil + } + + for _, editor := range []string{"editor", "sensible-editor", "vi", "nano"} { + if env.CommandInSearchPath(editor) { + return []string{editor}, nil + } + } + + return nil, errors.New("no local editor found; set 'EDITOR' or provide '--changelog-comment'") +} + +func prepareRenderMessage(env *azldev.Env, specPath string, update *locklessRenderUpdate) error { + if update.subjectProvided || update.subject != "" || env.DryRun() || (update.local && update.noGitCommit) { + return nil + } + + autorelease, err := locklessSpecUsesAutorelease(env, specPath) + if err != nil { + return err + } + + if autorelease && update.noGitCommit { + return nil + } + + template := renderChangelogTemplate + + switch { + case autorelease: + template = renderAutoreleaseTemplate + case update.local: + template = renderLocalCommitTemplate + case update.noGitCommit: + template = renderNoGitChangelogTemplate + } + + update.subject, err = editRenderMessage(env, specPath, prefillRenderChanges(template, update.changes, autorelease)) + + return err +} + +func prefillRenderChanges(template, changes string, autorelease bool) string { + if strings.TrimSpace(changes) == "" { + return template + } + + prefix := "\n" + if autorelease { + prefix += "...\n" + } + + changes = strings.TrimSuffix(changes, "\n") + + return prefix + "- " + strings.ReplaceAll(changes, "\n", "\n- ") + + "\n\n" + strings.TrimPrefix(template, "\n\n") +} + +func editRenderMessage(env *azldev.Env, specPath, template string) (string, error) { + if !renderIsTerminal(os.Stdin.Fd()) || !renderIsTerminal(os.Stdout.Fd()) { + return "", errors.New("non-interactive renders must use '--changelog-comment' and/or '--no-git-commit'") + } + + args, err := renderEditor(env) + if err != nil { + return "", err + } + + path, cleanup, err := renderMessageFile(env, template) + if err != nil { + return "", err + } + defer cleanup() + + slog.Info("Enter render message, save and close the editor; blank lines and lines starting with # are ignored", + "spec", specPath) + + raw := exec.CommandContext(env, args[0], append(args[1:], path)...) + if err := runRenderInteractive(env, raw); err != nil { + return "", fmt.Errorf("editing render message:\n%w", err) + } + + content, err := fileutils.ReadFile(env.FS(), path) + if err != nil { + return "", fmt.Errorf("reading edited render message:\n%w", err) + } + + message := cleanRenderChangelog(string(content)) + if message == "" { + return "", errors.New("no render message provided; rendering will not be done") + } + + return message, nil +} + +func cleanRenderChangelog(content string) string { + var lines []string + + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSuffix(line, "\r") + if strings.TrimSpace(line) != "" && !strings.HasPrefix(line, "#") { + lines = append(lines, line) + } + } + + return strings.Join(lines, "\n") +} + +func renderMessageFile(env *azldev.Env, content string) (path string, cleanup func(), err error) { + dir, err := fileutils.MkdirTempInTempDir(env.FS(), "azldev-render-message-") + if err != nil { + return "", nil, fmt.Errorf("creating render message directory:\n%w", err) + } + + cleanup = func() { + if err := env.FS().RemoveAll(dir); err != nil { + slog.Debug("Failed to remove render message directory", "path", dir, "error", err) + } + } + + path = filepath.Join(dir, "message.txt") + if err := fileutils.WriteFile(env.FS(), path, []byte(content), fileperms.PrivateFile); err != nil { + cleanup() + + return "", nil, fmt.Errorf("creating render message file:\n%w", err) + } + + return path, cleanup, nil +} + +func runRenderInteractive(env *azldev.Env, raw *exec.Cmd) error { + raw.Stdin = os.Stdin + raw.Stdout = os.Stdout + raw.Stderr = os.Stderr + + command, err := env.Command(raw) + if err != nil { + return fmt.Errorf("creating interactive render command:\n%w", err) + } + + if err := command.Run(env); err != nil { + return fmt.Errorf("running interactive render command:\n%w", err) + } + + return nil +} + +func runLocklessRenderCommit(env *azldev.Env, update *locklessRenderUpdate, paths []string) error { + if !update.subjectProvided && update.subject == "" { + return errors.New("no render message collected before Git commit") + } + + message := update.subject + if summary, body, multiline := strings.Cut(message, "\n"); multiline && !strings.HasPrefix(body, "\n") { + message = summary + "\n\n" + body + } + + args := []string{"commit", "--only", "-m", message} + if update.subjectProvided && update.changes != "" { + args = append(args, "-m", update.changes) + } + + args = append(args, "--") + args = append(args, paths...) + + if _, err := git.RunInDir(env, env, update.repoRoot, args...); err != nil { + return fmt.Errorf("running render commit:\n%w", err) + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/render_message_internal_test.go b/internal/app/azldev/cmds/component/render_message_internal_test.go new file mode 100644 index 000000000..b9a43898c --- /dev/null +++ b/internal/app/azldev/cmds/component/render_message_internal_test.go @@ -0,0 +1,464 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os" + "os/exec" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setRenderEditor(t *testing.T, env *testutils.TestEnv, editor string) { + t.Helper() + + original := renderIsTerminal + renderIsTerminal = func(_ uintptr) bool { return true } + + t.Cleanup(func() { renderIsTerminal = original }) + + osEnv, ok := env.TestOSEnv.(*testctx.TestOSEnv) + require.True(t, ok) + osEnv.SetEnv("EDITOR", editor) +} + +func TestRenderEditorSelection(t *testing.T) { + for _, testCase := range []struct { + name, editor, available, wantError string + want []string + }{ + { + name: "quoted arguments", editor: "edit-tool --wait 'two words'", available: "edit-tool", + want: []string{"edit-tool", "--wait", "two words"}, + }, + {name: "fallback", available: "vi", want: []string{"vi"}}, + {name: "invalid quoting", editor: "edit-tool '", wantError: "parsing 'EDITOR'"}, + {name: "unavailable configured editor", editor: "edit-tool", wantError: "unavailable"}, + {name: "no editor", wantError: "no local editor"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, testCase.editor) + + if testCase.available != "" { + env.CmdFactory.RegisterCommandInSearchPath(testCase.available) + } + + args, err := renderEditor(env.Env) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.NoError(t, err) + assert.Equal(t, testCase.want, args) + } + }) + } +} + +func TestRenderMessageRequiresInteractiveTTY(t *testing.T) { + for _, testCase := range []struct { + name string + stdinTTY, stdoutTTY bool + }{ + {name: "stdin is not a tty", stdoutTTY: true}, + {name: "stdout is not a tty", stdinTTY: true}, + {name: "neither is a tty"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "unavailable-editor") + + renderIsTerminal = func(descriptor uintptr) bool { + switch descriptor { + case os.Stdin.Fd(): + return testCase.stdinTTY + case os.Stdout.Fd(): + return testCase.stdoutTTY + default: + t.Fatalf("unexpected terminal descriptor: %d", descriptor) + + return false + } + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not invoke an editor without both TTYs: %v", cmd.Args) + + return nil + } + message, err := editRenderMessage(env.Env, "/working/curl.spec", renderLocalCommitTemplate) + require.ErrorContains(t, err, "non-interactive renders must use '--changelog-comment' and/or '--no-git-commit'") + assert.Empty(t, message) + }) + } +} + +func TestNonInteractiveRenderMessageOptions(t *testing.T) { + for _, testCase := range []struct { + name, release string + local, noGit, provided bool + wantError bool + }{ + {name: "provided upstream static", release: "1", provided: true}, + {name: "provided upstream autorelease", release: "%autorelease", provided: true}, + {name: "provided local", release: "1", local: true, provided: true}, + {name: "local static no commit", release: "1", local: true, noGit: true}, + {name: "local autorelease no commit", release: "%autorelease", local: true, noGit: true}, + {name: "upstream autorelease no commit", release: "%autorelease", noGit: true}, + {name: "upstream static no commit still needs message", release: "1", noGit: true, wantError: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + original := renderIsTerminal + renderIsTerminal = func(_ uintptr) bool { return false } + + t.Cleanup(func() { renderIsTerminal = original }) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + const specPath = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, + []byte(renderTestSpec("1", testCase.release, "")), fileperms.PublicFile)) + + bumped := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + + bumped = true + + return nil + } + + update := &locklessRenderUpdate{ + local: testCase.local, noGitCommit: testCase.noGit, subjectProvided: testCase.provided, + } + if testCase.provided { + update.subject = "Provided render message" + } + + _, err := finishLocklessSpec(env.Env, specPath, update, true, false) + if testCase.wantError { + require.ErrorContains(t, err, "non-interactive renders must use") + } else { + require.NoError(t, err) + } + + assert.Equal(t, !testCase.local && testCase.release == "1" && !testCase.wantError, bumped) + }) + } +} + +func TestFinishLocklessSpecMessages(t *testing.T) { + for _, testCase := range []struct { + name, release, message, template string + provided, local, noGit bool + }{ + {name: "upstream static commit", release: "1", template: renderChangelogTemplate}, + {name: "upstream static no commit", release: "1", noGit: true, template: renderNoGitChangelogTemplate}, + {name: "upstream autorelease commit", release: "%autorelease -b 9", template: renderAutoreleaseTemplate}, + {name: "upstream autorelease no commit", release: "%autorelease", noGit: true}, + {name: "local static commit", release: "9", local: true, template: renderLocalCommitTemplate}, + {name: "local static no commit", release: "9", local: true, noGit: true}, + {name: "local autorelease commit", release: "%autorelease", local: true, template: renderAutoreleaseTemplate}, + {name: "local autorelease no commit", release: "%autorelease", local: true, noGit: true}, + {name: "provided text", release: "1", message: "Explicit entry", provided: true}, + {name: "provided multiline text", release: "1", message: "# Explicit\n\nKeep details.\n", provided: true}, + {name: "explicit empty value", release: "1", provided: true}, + {name: "provided autorelease", release: "%autorelease", message: "Git only", provided: true}, + {name: "provided local", release: "9", local: true, message: "Git only", provided: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "edit-tool --wait") + env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + + wantBump := !testCase.local && !strings.Contains(testCase.release, "%autorelease") + if wantBump { + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + } + + const specPath = "/working/curl.spec" + + original := renderTestSpec("1", testCase.release, "* Preserve changelog\n- Existing entry\n") + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, []byte(original), fileperms.PublicFile)) + + editCalls, bumpCalls := 0, 0 + + wantMessage := testCase.message + if testCase.template != "" { + wantMessage = "Fix feature\nKeep details." + } + + var messagePath string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + switch cmd.Args[0] { + case "edit-tool": + editCalls++ + + assert.Equal(t, "--wait", cmd.Args[1]) + messagePath = cmd.Args[2] + assert.Same(t, os.Stdin, cmd.Stdin) + + content, err := fileutils.ReadFile(env.TestFS, messagePath) + require.NoError(t, err) + assert.Equal(t, testCase.template, string(content)) + + return fileutils.WriteFile(env.TestFS, messagePath, + []byte("Fix feature\n\nKeep details.\n"+string(content)), fileperms.PrivateFile) + case "rpmdev-bumpspec": + bumpCalls++ + + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", wantMessage, specPath}, cmd.Args) + + if testCase.template != "" { + assert.Equal(t, 1, editCalls, "collect the message before bumpspec") + } + default: + t.Fatalf("unexpected command: %v", cmd.Args) + } + + return nil + } + + update := &locklessRenderUpdate{ + subjectProvided: testCase.provided, local: testCase.local, noGitCommit: testCase.noGit, + } + if testCase.provided { + update.subject = testCase.message + } + + _, err := finishLocklessSpec(env.Env, specPath, update, true, false) + require.NoError(t, err) + assert.Equal(t, wantMessage, update.subject) + + if wantBump { + assert.Equal(t, 1, bumpCalls) + } else { + assert.Zero(t, bumpCalls) + } + + if testCase.template == "" { + assert.Zero(t, editCalls) + } else { + assert.Equal(t, 1, editCalls) + + exists, err := fileutils.Exists(env.TestFS, messagePath) + require.NoError(t, err) + assert.False(t, exists) + } + + content, err := fileutils.ReadFile(env.TestFS, specPath) + require.NoError(t, err) + assert.Equal(t, original, string(content)) + }) + } +} + +func TestCleanRenderChangelog(t *testing.T) { + for _, testCase := range []struct { + name string + lines []string + want string + crlf bool + }{ + { + name: "comments and blank lines", + lines: []string{"", "", "# Instructions", "Fix feature", " \t ", "Keep details.", "# Comment", ""}, + want: "Fix feature\nKeep details.", + }, + { + name: "preserve message whitespace and hashes", + lines: []string{" # Indented message", "Keep # inline hash "}, + want: " # Indented message\nKeep # inline hash ", + }, + {name: "only ignored lines", lines: []string{"", "\t", "# Instructions", "#", ""}}, + { + name: "CRLF", lines: []string{"", "# Instructions", "Fix feature", "", "Keep details.", ""}, + want: "Fix feature\nKeep details.", crlf: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + content := strings.Join(testCase.lines, "\n") + if testCase.crlf { + content = strings.ReplaceAll(content, "\n", "\r\n") + } + + assert.Equal(t, testCase.want, cleanRenderChangelog(content)) + }) + } +} + +func TestFinishLocklessSpecRejectsEmptyAndFailedEditor(t *testing.T) { + editorErr := errors.New("editor failed") + + for _, testCase := range []struct { + name, response string + fail bool + }{ + {name: "blank"}, + {name: "only blank lines", response: "\n \t\n"}, + {name: "only comments and blank lines", response: "\n# Instructions\n#\n \t\n"}, + {name: "failure", fail: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "edit-tool") + env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + + const specPath = "/working/curl.spec" + + original := renderTestSpec("1", "9", "") + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, []byte(original), fileperms.PublicFile)) + + var messagePath string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.Equal(t, "edit-tool", cmd.Args[0], "must fail before bumpspec or Git") + messagePath = cmd.Args[1] + + if testCase.fail { + return editorErr + } + + return fileutils.WriteFile(env.TestFS, messagePath, []byte(testCase.response), fileperms.PrivateFile) + } + + _, err := finishLocklessSpec(env.Env, specPath, + &locklessRenderUpdate{changes: "abc123 Upstream fix"}, true, false) + if testCase.fail { + require.ErrorIs(t, err, editorErr) + } else { + require.ErrorContains(t, err, "no render message provided") + } + + exists, err := fileutils.Exists(env.TestFS, messagePath) + require.NoError(t, err) + assert.False(t, exists) + + content, err := fileutils.ReadFile(env.TestFS, specPath) + require.NoError(t, err) + assert.Equal(t, original, string(content)) + }) + } +} + +func TestLocklessRenderCommitRejectsMissingMessage(t *testing.T) { + env := testutils.NewTestEnv(t) + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not open Git's editor: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("must not run Git without a message: %v", cmd.Args) + + return "", nil + } + require.ErrorContains(t, runLocklessRenderCommit(env.Env, + &locklessRenderUpdate{repoRoot: "/project", changes: "abc123 Upstream fix"}, + []string{"SPECS/c/curl", ":(top,literal)inputs.toml"}), "no render message collected") +} + +func TestLocklessRenderCommitReusesMessage(t *testing.T) { + for _, message := range []string{"Summary\nBody", "Summary\n\nBody"} { + t.Run(message, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("must not open Git's editor: %v", cmd.Args) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", "Summary\n\nBody", + "--", "SPECS/c/curl", + }, cmd.Args) + + return "", nil + } + require.NoError(t, runLocklessRenderCommit(env.Env, &locklessRenderUpdate{ + repoRoot: "/project", subject: message, changes: "abc123 Upstream fix", + }, []string{"SPECS/c/curl"})) + }) + } +} + +func TestRenderMessagePrefillsUpstreamChanges(t *testing.T) { + const autorelease = "%autorelease" + + for _, testCase := range []struct { + name, release, changes, prefix, message string + }{ + { + name: "static", release: "1", changes: "abc123 First fix\ndef456 Second fix", + prefix: "\n- abc123 First fix\n- def456 Second fix\n\n", + message: "Summary\n- abc123 First fix\n- def456 Second fix", + }, + { + name: "autorelease", release: autorelease, changes: "abc123 First fix\ndef456 Second fix\n", + prefix: "\n...\n- abc123 First fix\n- def456 Second fix\n\n", + message: "Summary\n...\n- abc123 First fix\n- def456 Second fix", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setRenderEditor(t, env, "edit-tool") + env.CmdFactory.RegisterCommandInSearchPath("edit-tool") + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + const specPath = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, specPath, + []byte(renderTestSpec("1", testCase.release, "")), fileperms.PublicFile)) + + editorCalls := 0 + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if cmd.Args[0] == "rpmdev-bumpspec" { + assert.Equal(t, []string{"rpmdev-bumpspec", "-c", testCase.message, specPath}, cmd.Args) + + return nil + } + + require.Equal(t, "edit-tool", cmd.Args[0]) + + editorCalls++ + content, err := fileutils.ReadFile(env.TestFS, cmd.Args[1]) + require.NoError(t, err) + + template := renderChangelogTemplate + if testCase.release == autorelease { + template = renderAutoreleaseTemplate + } + + assert.Equal(t, testCase.prefix+strings.TrimPrefix(template, "\n\n"), string(content)) + + return fileutils.WriteFile(env.TestFS, cmd.Args[1], []byte("Summary"+string(content)), + fileperms.PrivateFile) + } + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + summary, body, _ := strings.Cut(testCase.message, "\n") + assert.Equal(t, []string{ + "git", "-C", "/project", "commit", "--only", "-m", summary + "\n\n" + body, + "--", "SPECS/c/curl", + }, cmd.Args) + + return "", nil + } + + update := &locklessRenderUpdate{repoRoot: "/project", changes: testCase.changes} + _, err := finishLocklessSpec(env.Env, specPath, update, true, false) + require.NoError(t, err) + assert.Equal(t, testCase.message, update.subject) + require.NoError(t, runLocklessRenderCommit(env.Env, update, []string{"SPECS/c/curl"})) + assert.Equal(t, 1, editorCalls) + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_modes_internal_test.go b/internal/app/azldev/cmds/component/render_modes_internal_test.go new file mode 100644 index 000000000..34ac01eaa --- /dev/null +++ b/internal/app/azldev/cmds/component/render_modes_internal_test.go @@ -0,0 +1,509 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "context" + "errors" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/lockfile" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestReadLocklessComponents(t *testing.T) { + for _, testCase := range []struct { + name string + lines []string + missing bool + want map[string]bool + }{ + {name: "missing file", missing: true, want: map[string]bool{}}, + {name: "empty file", want: map[string]bool{}}, + { + name: "exact names with whitespace and duplicates", + lines: []string{" rust-asyncband \r", "", "\tcurl\t", "rust-asyncband", "Curl"}, + want: map[string]bool{"rust-asyncband": true, "curl": true, "Curl": true}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/subdir/lockless-components.txt", []byte("wrong-component"), fileperms.PublicFile, + )) + require.NoError(t, env.TestOSEnv.Chdir("/project/subdir")) + + if !testCase.missing { + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/lockless-components.txt", + []byte(strings.Join(testCase.lines, "\n")), fileperms.PublicFile, + )) + } + + names, err := projectconfig.ReadLocklessComponents(env.TestFS, env.Env.ProjectDir()) + require.NoError(t, err) + assert.Equal(t, testCase.want, names) + }) + } + + t.Run("read errors are not treated as an empty list", func(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.MkdirAll(env.TestFS, "/project/lockless-components.txt")) + + _, err := projectconfig.ReadLocklessComponents(env.TestFS, env.Env.ProjectDir()) + require.ErrorContains(t, err, "lockless-components.txt") + }) +} + +type renderModeObservingPreparer struct { + sources.SourcePreparer + observe func(components.Component) + err error + prepare func(components.Component, string) error +} + +func (p *renderModeObservingPreparer) PrepareSources( + _ context.Context, component components.Component, dir string, _ bool, +) error { + p.observe(component) + + if p.prepare != nil { + return p.prepare(component, dir) + } + + return p.err +} + +func TestRenderComponentsRequiresCleanLocklessRepository(t *testing.T) { + statusErr := errors.New("git status failed") + + for _, testCase := range []struct { + name, status, wantError string + allow, checkOnly, noGitCommit, mixed bool + statusErr error + }{ + {name: "clean"}, + {name: "unstaged tracked", status: " M tracked\n", wantError: "uncommitted or untracked files"}, + {name: "staged tracked", status: "M tracked\n", wantError: "uncommitted or untracked files"}, + {name: "untracked", status: "?? scratch\n", wantError: "uncommitted or untracked files"}, + {name: "explicit override", status: "?? scratch\n", allow: true}, + { + name: "check-only still checks", status: " M tracked\n", + checkOnly: true, wantError: "uncommitted or untracked files", + }, + { + name: "no-git-commit still checks", status: " M tracked\n", + noGitCommit: true, wantError: "uncommitted or untracked files", + }, + { + name: "mixed selection stops before preparation", status: "?? scratch\n", + mixed: true, wantError: "uncommitted or untracked files", + }, + {name: "status failure", statusErr: statusErr, wantError: "checking project repository"}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + if testCase.mixed { + env = testutils.NewTestEnv(t) + env.AddUpstreamComponent(t, "bash") + env.Config.LocklessComponents = map[string]bool{"curl": true} + } + + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeUpstream, UpstreamCommit: "configured-commit", + }, + } + queried := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + scope := []string{":/", ":(exclude,literal)specs"} + if queried { + scope = []string{":(literal)specs/c/curl"} + } + + queried = true + + assert.Equal(t, append([]string{ + "git", "-C", "/project", "status", "--porcelain=v1", + "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...), cmd.Args) + + return testCase.status, testCase.statusErr + } + started := false + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + started = true + + return &renderModeObservingPreparer{ + observe: func(_ components.Component) {}, + err: errors.New("stop after preflight"), + }, nil + } + + t.Cleanup(func() { newSourcePreparer = original }) + + results, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/project/SPECS", + OutputDirExplicit: true, + AllowUncommitted: testCase.allow, + CheckOnly: testCase.checkOnly, + NoGitCommit: testCase.noGitCommit, + }) + assert.Equal(t, !testCase.allow, queried) + + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + assert.Empty(t, results) + assert.False(t, started, "refuse before fetching any selected component") + + if testCase.statusErr != nil { + require.ErrorIs(t, err, statusErr) + } else { + assert.Contains(t, err.Error(), "--allow-uncommitted") + } + } else { + require.NoError(t, err) + assert.True(t, started) + require.Len(t, results, 1) + assert.Contains(t, results[0].Error, "stop after preflight") + } + }) + } +} + +func TestLocklessRenderWorktreeCheckScope(t *testing.T) { + for _, testCase := range []struct { + name string + source projectconfig.SpecSourceType + lockless bool + checked bool + }{ + {name: "locked upstream", source: projectconfig.SpecSourceTypeUpstream}, + {name: "locked local", source: projectconfig.SpecSourceTypeLocal}, + {name: "lockless local", source: projectconfig.SpecSourceTypeLocal, lockless: true, checked: true}, + {name: "lockless upstream", source: projectconfig.SpecSourceTypeUpstream, lockless: true, checked: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"curl": testCase.lockless, "rust-asyncband": true} + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: testCase.source}, + }) + + queried := false + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + queried = true + + return "?? scratch\n", nil + } + err := validateRenderComponents(env.Env, []components.Component{comp}, false) + + assert.Equal(t, testCase.checked, queried) + + if testCase.checked { + require.ErrorContains(t, err, "uncommitted or untracked files") + } else { + require.NoError(t, err) + } + }) + } +} + +func TestLocklessRenderWorktreeConfiguredScopes(t *testing.T) { + statusErr := errors.New("component status failed") + + for _, testCase := range []struct { + name, specDir, wantRoot string + outsideStatus, componentStatus string + componentError, wantError bool + }{ + {name: "default rendered root"}, + {name: "relative rendered root", specDir: "generated/specs", wantRoot: "generated/specs"}, + {name: "absolute rendered root", specDir: "/project/generated/specs", wantRoot: "generated/specs"}, + {name: "parent rendered root", specDir: "/specs", wantRoot: "../specs"}, + {name: "normalized rendered root", specDir: "generated/../specs/"}, + {name: "literal rendered root", specDir: "generated/[specs]*", wantRoot: "generated/[specs]*"}, + {name: "changes outside rendered root", outsideStatus: " M input.toml\n", wantError: true}, + {name: "selected component modified", componentStatus: " M specs/c/Curl/Curl.spec\n", wantError: true}, + {name: "selected component staged", componentStatus: "M specs/c/Curl/Curl.spec\n", wantError: true}, + {name: "selected component untracked", componentStatus: "?? specs/c/Curl/new.patch\n", wantError: true}, + { + name: "another selected component modified", + componentStatus: " M specs/r/rust-asyncband/rust-asyncband.spec\n", wantError: true, + }, + {name: "selected component status failure", componentError: true, wantError: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = testCase.specDir + env.Config.LocklessComponents = map[string]bool{"Curl": true} + ctrl := gomock.NewController(t) + comps := make([]components.Component, 0, 2) + + for _, name := range []string{"Curl", "rust-asyncband"} { + comp := components_testutils.NewMockComponent(ctrl) + comp.EXPECT().GetName().AnyTimes().Return(name) + comps = append(comps, comp) + } + + root := testCase.wantRoot + if root == "" { + root = "specs" + } + + calls := 0 + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + calls++ + require.LessOrEqual(t, calls, 2) + + scope := []string{":/", ":(exclude,literal)" + root} + status := testCase.outsideStatus + + if calls == 2 { + scope = []string{":(literal)" + root + "/c/Curl", ":(literal)" + root + "/r/rust-asyncband"} + status = testCase.componentStatus + } + + assert.Equal(t, append([]string{ + "git", "-C", "/project", "status", "--porcelain=v1", + "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...), cmd.Args) + + if calls == 2 && testCase.componentError { + return "", statusErr + } + + return status, nil + } + + err := validateRenderComponents(env.Env, comps, false) + if testCase.wantError { + require.Error(t, err) + + if testCase.componentError { + require.ErrorIs(t, err, statusErr) + } else { + assert.Contains(t, err.Error(), "--allow-uncommitted") + } + } else { + require.NoError(t, err) + } + + wantCalls := 2 + if testCase.outsideStatus != "" { + wantCalls = 1 + } + + assert.Equal(t, wantCalls, calls) + }) + } +} + +func TestLocklessRenderPreservesAllFilesWithoutMock(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + version := env.Config.Distros["test-distro"].Versions["1.0"] + version.MockConfigPath = "" + env.Config.Distros["test-distro"].Versions["1.0"] = version + env.Config.Components["curl"] = projectconfig.ComponentConfig{ + Name: "curl", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeLocal}, + } + checked := false + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + scope := []string{":/", ":(exclude,literal)specs"} + if checked { + scope = []string{":(literal)specs/c/curl"} + } + + checked = true + + assert.Equal(t, append([]string{ + "git", "-C", "/project", "status", "--porcelain=v1", + "--untracked-files=all", "--ignore-submodules=none", "--", + }, scope...), cmd.Args) + + return "", nil + } + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + t.Fatalf("unexpected command during host-only rendering: %v", cmd.Args) + + return nil + } + + const ( + rawSpec = "Name: curl\nVersion: 1\nRelease: %autorelease\n\n%changelog\n%autochangelog\n" + changelog = "* Old entry\n- Keep unchanged\n" + ) + + files := map[string]string{ + "curl.spec": rawSpec, "changelog": changelog, "fix.patch": "patch", + "unreferenced": "keep", ".gitignore": "upstream ignore rules", + "extras/nested.txt": "unreferenced nested file", + } + + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, fs opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return &renderModeObservingPreparer{ + observe: func(_ components.Component) {}, + prepare: func(_ components.Component, dir string) error { + for name, content := range files { + require.NoError(t, fileutils.WriteFile(fs, filepath.Join(dir, name), + []byte(content), fileperms.PublicFile)) + } + + return fileutils.WriteFile(fs, filepath.Join(dir, ".git/HEAD"), + []byte("temporary history"), fileperms.PublicFile) + }, + }, nil + } + + t.Cleanup(func() { newSourcePreparer = original }) + + results, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/rendered", OutputDirExplicit: true, FailOnError: true, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, renderStatusOK, results[0].Status) + + for name, want := range files { + content, err := fileutils.ReadFile(env.TestFS, filepath.Join("/rendered/c/curl", name)) + require.NoError(t, err) + assert.Equal(t, want, string(content)) + } + + exists, err := fileutils.Exists(env.TestFS, "/rendered/c/curl/.git") + require.NoError(t, err) + assert.False(t, exists) +} + +func TestRenderComponentsSelectsLockModeFromFile(t *testing.T) { + for _, testCase := range []struct { + name string + withoutLockfile bool + }{ + {name: "project component list"}, + {name: "internal whole-project override", withoutLockfile: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + if testCase.withoutLockfile { + env = testutils.NewTestEnvWithoutLockfile(t) + env.Config.Project.LockDir = "/project/locks" + } + + for _, name := range []string{"curl", "rust-asyncband"} { + env.Config.Components[name] = projectconfig.ComponentConfig{ + Name: name, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeUpstream, + UpstreamCommit: "configured-commit", + }, + } + } + + lock := lockfile.New() + lock.UpstreamCommit = "configured-commit" + env.WriteLock(t, "curl", lock) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/locks/rust-asyncband.lock", []byte("corrupt lock"), fileperms.PublicFile, + )) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/project/lockless-components.txt", []byte("rust-asyncband\n"), fileperms.PublicFile, + )) + names, err := projectconfig.ReadLocklessComponents(env.TestFS, env.Env.ProjectDir()) + require.NoError(t, err) + + env.Config.LocklessComponents = names + + var observedMutex sync.Mutex + + observed := make(map[string]*projectconfig.ComponentLockData) + observedErr := errors.New("stop after observing render mode") + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return &renderModeObservingPreparer{ + observe: func(comp components.Component) { + observedMutex.Lock() + defer observedMutex.Unlock() + + observed[comp.GetName()] = comp.GetConfig().Locked + }, + err: observedErr, + }, nil + } + + t.Cleanup(func() { newSourcePreparer = original }) + + results, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/rendered", + OutputDirExplicit: true, + }) + require.NoError(t, err) + require.Len(t, results, 2) + require.Len(t, observed, 2) + + if testCase.withoutLockfile { + assert.Nil(t, observed["curl"], "the internal override applies to every component") + } else { + require.NotNil(t, observed["curl"]) + assert.Equal(t, "configured-commit", observed["curl"].UpstreamCommit) + } + + assert.Nil(t, observed["rust-asyncband"]) + assert.Equal(t, testCase.withoutLockfile, env.Env.WithoutLockfile()) + }) + } +} + +func TestRenderComponentsRequiresLocksForUnlistedComponents(t *testing.T) { + env := testutils.NewTestEnv(t) + env.AddUpstreamComponent(t, "curl") + env.Config.Components["rust-asyncband"] = projectconfig.ComponentConfig{ + Name: "rust-asyncband", + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + } + require.NoError(t, fileutils.WriteFile( + env.TestFS, filepath.Join(env.Env.ProjectDir(), "lockless-components.txt"), []byte("curl\n"), fileperms.PublicFile, + )) + env.Config.LocklessComponents = map[string]bool{"curl": true} + + _, err := RenderComponents(env.Env, &RenderOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + OutputDir: "/rendered", + OutputDirExplicit: true, + }) + require.ErrorContains(t, err, "lock file validation failed") +} diff --git a/internal/app/azldev/cmds/component/render_output_internal_test.go b/internal/app/azldev/cmds/component/render_output_internal_test.go new file mode 100644 index 000000000..135249a96 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_output_internal_test.go @@ -0,0 +1,700 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +const ( + testRenderOutputDir = "/project/SPECS/c/curl" + testRenderStagingDir = "/staging/curl" + renderTestDiffCommand = "diff" + renderTestResetCommand = "reset" + renderTestCheckoutCommand = "checkout" + renderTestOverwrite = "overwrite" + renderTestSpecDiff = "SPECS/c/curl/curl.spec\n" + renderTestAutorelease = "%autorelease -b 9" +) + +func TestLocklessRenderNEVRBeforeReplacement(t *testing.T) { + for _, testCase := range []struct { + name, existing, rendered, newName, rawRelease string + reject, queryFails, toolMissing bool + }{ + {name: "numeric release increase", existing: "1-9", rendered: "1-10"}, + {name: "equal", existing: "1-9", rendered: "1-9", reject: true}, + {name: "autorelease equal", existing: "1-9", rendered: "1-9", rawRelease: renderTestAutorelease}, + {name: "autorelease increase", existing: "1-9", rendered: "1-10", rawRelease: "%autorelease -b 10"}, + { + name: "autorelease release downgrade", existing: "1-10", rendered: "1-9", + rawRelease: renderTestAutorelease, reject: true, + }, + { + name: "autorelease version downgrade", existing: "2-1", rendered: "1-9", + rawRelease: renderTestAutorelease, reject: true, + }, + { + name: "autorelease name change", existing: "1-9", rendered: "1-9", + rawRelease: renderTestAutorelease, newName: "other", reject: true, + }, + {name: "version downgrade", existing: "2-1", rendered: "1-99", reject: true}, + {name: "epoch increase", existing: "9-9", rendered: "1:1-1"}, + {name: "epoch decrease", existing: "1:1-1", rendered: "9-9", reject: true}, + {name: "prerelease downgrade", existing: "1-1", rendered: "1~rc1-2", reject: true}, + {name: "name change", existing: "1-1", rendered: "2-1", newName: "other", reject: true}, + {name: "query failure", existing: "1-1", rendered: "2-1", queryFails: true, reject: true}, + {name: "missing rpmspec", existing: "1-1", rendered: "2-1", toolMissing: true, reject: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, true) + + if !testCase.toolMissing { + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + } + + prep := &preparedComponent{ + lockless: true, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + } + existing, err := rpm.NewVersion(testCase.existing) + require.NoError(t, err) + rendered, err := rpm.NewVersion(testCase.rendered) + require.NoError(t, err) + + rawRelease := rendered.Release() + if testCase.rawRelease != "" { + rawRelease = testCase.rawRelease + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/staging/curl/curl.spec", + []byte(renderTestSpec(rendered.Version(), rawRelease, "")), fileperms.PublicFile)) + + newName := "curl" + if testCase.newName != "" { + newName = testCase.newName + } + + history := &fakeRenderNEVRHistory{rendered: &rpm.SpecInfo{Name: newName, Version: *rendered}} + opened := stubRenderNEVRHistory(t, history) + + queries := 0 + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + queries++ + + assert.False(t, prep.outputTouched, "NEVR checks must precede removal and recovery state") + assertOriginalRenderOutput(t, env.TestFS, true) + + dir := testRenderOutputDir + + name, version := "curl", existing + if queries == 2 { + dir, name, version = testRenderStagingDir, newName, rendered + } + + assert.Equal(t, dir, cmd.Dir) + assert.Equal(t, []string{ + "rpmspec", "-q", "--srpm", "--define", "_sourcedir " + dir, + "--define", "_specdir " + dir, "--queryformat", rpm.SrpmQueryFormat, + filepath.Join(dir, "curl.spec"), + }, cmd.Args) + + if testCase.queryFails { + return "", errors.New("rpmspec failed") + } + + return "name=" + name + "\nepoch=" + strconv.Itoa(version.Epoch()) + + "\nversion=" + version.Version() + "\nrelease=" + version.Release() + "\n", nil + } + + err = moveRenderedOutput(env.Env, testRenderStagingDir, prep, true) + if testCase.reject { + require.ErrorIs(t, err, errRenderNEVRValidation) + assertOriginalRenderOutput(t, env.TestFS, true) + assert.False(t, prep.outputTouched) + + if !testCase.queryFails && !testCase.toolMissing { + if testCase.rawRelease != "" { + require.ErrorContains(t, err, "new NEVR must be equal to or higher than existing NEVR") + } + + require.ErrorContains(t, err, "existing `curl-"+testCase.existing+"`") + require.ErrorContains(t, err, "new `"+newName+"-"+testCase.rendered+"`") + } + + for _, failOnError := range []bool{false, true} { + require.ErrorIs(t, checkRenderErrors([]*RenderResult{{ + Component: "curl", Status: renderStatusError, renderErr: err, + }}, failOnError), errRenderNEVRValidation) + } + } else { + require.NoError(t, err) + assert.True(t, prep.outputTouched) + + content, readErr := fileutils.ReadFile(env.TestFS, filepath.Join(testRenderOutputDir, "sidecar")) + require.NoError(t, readErr) + assert.Equal(t, "new sidecar", string(content)) + } + + switch { + case testCase.toolMissing: + assert.Zero(t, queries) + case testCase.queryFails: + assert.Equal(t, 1, queries) + case testCase.rawRelease != "": + // Only the existing spec uses rpmspec; history resolves the new %autorelease. + assert.Equal(t, 1, queries) + assert.Equal(t, 1, *opened) + assert.Equal(t, testRenderStagingDir, history.preparedDir) + assert.True(t, history.removed) + default: + assert.Equal(t, 2, queries) + assert.Zero(t, *opened) + } + }) + } +} + +func TestRenderNEVRSkipsFirstAndLockedRenders(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, autorelease := range []bool{false, true} { + t.Run(strconv.FormatBool(existing)+"/"+strconv.FormatBool(autorelease), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + + if autorelease { + require.NoError(t, fileutils.WriteFile(env.TestFS, "/staging/curl/curl.spec", + []byte(renderTestSpec("1", renderTestAutorelease, "")), fileperms.PublicFile)) + } + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + t.Fatalf("must not query NEVR for first or locked output: %v", cmd.Args) + + return "", nil + } + require.NoError(t, moveRenderedOutput(env.Env, testRenderStagingDir, &preparedComponent{ + lockless: !existing, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + }, true)) + }) + } + } +} + +func TestMoveRenderedOutput_LockModes(t *testing.T) { + for _, lockless := range []bool{false, true} { + for _, crossDevice := range []bool{false, true} { + t.Run(strconv.FormatBool(lockless)+"/"+strconv.FormatBool(crossDevice), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, false) + + outputFS := &renderOutputFaultFS{Fs: env.TestFS} + if crossDevice { + outputFS.failure = "cross-device" + } + + env.TestFS = outputFS + require.NoError(t, outputFS.Chmod("/staging/curl/sidecar", fileperms.PublicExecutable)) + + err := moveRenderedOutput(env.Env, "/staging/curl", &preparedComponent{ + compOutputDir: testRenderOutputDir, lockless: lockless, + }, true) + if crossDevice && lockless { + require.ErrorIs(t, err, syscall.EXDEV) + + return + } + + require.NoError(t, err) + + path := filepath.Join(testRenderOutputDir, "sidecar") + content, err := fileutils.ReadFile(outputFS, path) + require.NoError(t, err) + assert.Equal(t, "new sidecar", string(content)) + + info, err := outputFS.Stat(path) + require.NoError(t, err) + assert.Equal(t, fileperms.PublicExecutable, info.Mode().Perm()) + + exists, err := fileutils.Exists(outputFS, "/staging/curl/sidecar") + require.NoError(t, err) + assert.Equal(t, !lockless, exists) + exists, err = fileutils.Exists(outputFS, "/staging/curl/.git/HEAD") + require.NoError(t, err) + assert.Equal(t, lockless, exists) + }) + } + } +} + +type fakeRenderNEVRHistory struct { + committed, rendered *rpm.SpecInfo + preparedDir string + removed bool +} + +func (h *fakeRenderNEVRHistory) Committed(string) (*rpm.SpecInfo, error) { + if h.committed == nil { + return nil, errors.New("unexpected committed history query") + } + + return h.committed, nil +} + +func (h *fakeRenderNEVRHistory) Rendered(preparedDir, _ string) (*rpm.SpecInfo, error) { + h.preparedDir = preparedDir + + return h.rendered, nil +} + +func (h *fakeRenderNEVRHistory) Remove() error { + h.removed = true + + return nil +} + +// stubRenderNEVRHistory replaces dist-git history resolution; callers must not run in parallel. +func stubRenderNEVRHistory(t *testing.T, history renderNEVRHistory) *int { + t.Helper() + + opened := 0 + original := openRenderNEVRHistory + openRenderNEVRHistory = func(_ *azldev.Env, parent, outputDir string) (renderNEVRHistory, error) { + opened++ + + assert.Equal(t, testRenderOutputDir, outputDir) + assert.Equal(t, "/staging", parent) + + return history, nil + } + + t.Cleanup(func() { openRenderNEVRHistory = original }) + + return &opened +} + +func testSpecInfo(t *testing.T, evr string) *rpm.SpecInfo { + t.Helper() + + version, err := rpm.NewVersion(evr) + require.NoError(t, err) + + return &rpm.SpecInfo{Name: "curl", Version: *version} +} + +func stubIncreasingRenderNEVR(t *testing.T, env *testutils.TestEnv) { + t.Helper() + + originalHistory := openRenderNEVRHistory + openRenderNEVRHistory = func(*azldev.Env, string, string) (renderNEVRHistory, error) { + return &fakeRenderNEVRHistory{ + committed: testSpecInfo(t, "1-1"), rendered: testSpecInfo(t, "1-2"), + }, nil + } + + t.Cleanup(func() { openRenderNEVRHistory = originalHistory }) + + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + original := env.CmdFactory.RunAndGetOutputHandler + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[0] == "rpmspec" { + release := "1" + if cmd.Dir == testRenderStagingDir { + release = "2" + } + + return "name=curl\nepoch=0\nversion=1\nrelease=" + release + "\n", nil + } + + if original != nil { + return original(cmd) + } + + t.Fatalf("unexpected command: %v", cmd.Args) + + return "", nil + } +} + +func writeOriginalRenderOutput(t *testing.T, fileSystem afero.Fs, root string) { + t.Helper() + + for name, content := range map[string]string{ + "curl.spec": "old spec", "extras/stale.patch": "old sidecar", ".no-change-rebuild": "old timestamp", + } { + require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join(root, name), + []byte(content), fileperms.PublicFile)) + } +} + +func setupRenderOutput(t *testing.T, fileSystem afero.Fs, existing bool) { + t.Helper() + + for name, content := range map[string]string{ + "curl.spec": "Name: curl\nVersion: 1\nRelease: 1\n", "sidecar": "new sidecar", ".git/HEAD": "upstream", + } { + require.NoError(t, fileutils.WriteFile(fileSystem, filepath.Join(testRenderStagingDir, name), + []byte(content), fileperms.PublicFile)) + } + + if existing { + writeOriginalRenderOutput(t, fileSystem, testRenderOutputDir) + writeOriginalRenderOutput(t, fileSystem, "/before") + } +} + +func assertOriginalRenderOutput(t *testing.T, fileSystem afero.Fs, existing bool) { + t.Helper() + + exists, err := fileutils.DirExists(fileSystem, testRenderOutputDir) + require.NoError(t, err) + assert.Equal(t, existing, exists) + + if existing { + diff, err := dirdiff.DiffDirs(fileSystem, "/before", testRenderOutputDir) + require.NoError(t, err) + assert.Empty(t, diff.Files) + } +} + +func mockRenderRecovery(t *testing.T, env *testutils.TestEnv) *[]string { + t.Helper() + + var commands []string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + exists, err := fileutils.Exists(env.TestFS, testRenderOutputDir) + require.NoError(t, err) + assert.False(t, exists, "failed output must be cleared before Git recovery") + + command := cmd.Args[3] + commands = append(commands, command) + + switch command { + case renderTestResetCommand: + assert.Equal(t, []string{"git", "-C", "/project", renderTestResetCommand, "--quiet", "--", "SPECS/c/curl"}, cmd.Args) + case renderTestCheckoutCommand: + assert.Equal(t, []string{"git", "-C", "/project", renderTestCheckoutCommand, "HEAD", "--", "SPECS/c/curl"}, cmd.Args) + writeOriginalRenderOutput(t, env.TestFS, testRenderOutputDir) + default: + require.FailNow(t, "unexpected Git command", "%v", cmd.Args) + } + + return "", nil + } + + return &commands +} + +func TestMoveLocklessOutput_PublicationFailure(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, failure := range []string{"processing", renderTestOverwrite, "remove", "move"} { + if (failure == renderTestOverwrite || failure == "remove") && !existing { + continue + } + + t.Run(failure+"/"+strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + env.TestFS = &renderOutputFaultFS{Fs: env.TestFS, failure: failure} + commands := mockRenderRecovery(t, env) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + prep := &preparedComponent{ + lockless: true, comp: comp, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + } + + mockResult := &sources.ComponentMockResult{} + if failure == "processing" { + mockResult.Error = errors.New("injected processing failure") + } + + stubIncreasingRenderNEVR(t, env) + result := finishOneComponent(env.Env, prep, + map[string]*sources.ComponentMockResult{"curl": mockResult}, + "/staging", failure != renderTestOverwrite, false, false) + require.Error(t, result.renderErr) + commitLocklessRenders(env.Env, []*preparedComponent{prep}, []*RenderResult{result}, false) + writeFailureMarkers(env.TestFS, []*RenderResult{result}, true, false) + assertOriginalRenderOutput(t, env.TestFS, existing) + + switch { + case failure == "processing" || failure == renderTestOverwrite: + assert.False(t, prep.outputTouched) + assert.Empty(t, *commands) + case existing: + assert.Equal(t, []string{renderTestResetCommand, renderTestCheckoutCommand}, *commands) + default: + assert.Equal(t, []string{renderTestResetCommand}, *commands) + } + }) + } + } +} + +func TestCommitLocklessRenders_RestoresFailedOutput(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, failure := range []string{"add", renderTestDiffCommand, "commit", ""} { + t.Run(failure+"/"+strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + prepared := []*preparedComponent{{ + lockless: true, comp: comp, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + update: &locklessRenderUpdate{ + repoRoot: "/project", path: "SPECS/c/curl", subject: "updated curl", + }, + }} + + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunHandler = func(_ *exec.Cmd) error { return nil } + stubIncreasingRenderNEVR(t, env) + result := finishOneComponent(env.Env, prepared[0], + map[string]*sources.ComponentMockResult{"curl": {}}, "/staging", true, false, false) + require.NoError(t, result.renderErr) + + results := []*RenderResult{result} + commands := mockRenderRecovery(t, env) + recoverCommand := env.CmdFactory.RunAndGetOutputHandler + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == failure { + return "", errors.New("injected Git failure") + } + + if cmd.Args[3] == renderTestDiffCommand { + return renderTestSpecDiff, nil + } + + if cmd.Args[3] == renderTestResetCommand || cmd.Args[3] == renderTestCheckoutCommand { + return recoverCommand(cmd) + } + + return "", nil + } + + commitLocklessRenders(env.Env, prepared, results, false) + writeFailureMarkers(env.TestFS, results, true, false) + + if failure != "" { + assert.Equal(t, renderStatusError, results[0].Status) + assert.Contains(t, results[0].Error, "injected Git failure") + assertOriginalRenderOutput(t, env.TestFS, existing) + + if existing { + assert.Equal(t, []string{renderTestResetCommand, renderTestCheckoutCommand}, *commands) + } else { + assert.Equal(t, []string{renderTestResetCommand}, *commands) + } + } else { + assert.Equal(t, renderStatusOK, results[0].Status) + assert.Empty(t, *commands) + + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(testRenderOutputDir, "curl.spec")) + require.NoError(t, err) + assert.Equal(t, "Name: curl\nVersion: 1\nRelease: 1\n", string(content)) + + for _, name := range []string{".git", "extras", renderErrorMarkerFile} { + exists, err := fileutils.Exists(env.TestFS, filepath.Join(testRenderOutputDir, name)) + require.NoError(t, err) + assert.False(t, exists) + } + } + }) + } + } +} + +func TestCommitLocklessRenders_ReportsRecoveryFailure(t *testing.T) { + for _, failure := range []string{renderTestResetCommand, renderTestCheckoutCommand} { + t.Run(failure, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, true) + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if cmd.Args[3] == failure { + return "", errors.New("injected recovery failure") + } + + return "", nil + } + prepared := []*preparedComponent{{ + lockless: true, outputTouched: true, outputExisted: true, compOutputDir: testRenderOutputDir, + }} + results := []*RenderResult{{Status: renderStatusError, Error: "original render failure", lockless: true}} + commitLocklessRenders(env.Env, prepared, results, false) + assert.Contains(t, results[0].Error, "original render failure") + assert.Contains(t, results[0].Error, "injected recovery failure") + assert.Equal(t, renderStatusError, results[0].Status) + }) + } +} + +func TestWriteFailureMarkers_LocklessPreservesOutput(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, checkOnly := range []bool{false, true} { + t.Run(strconv.FormatBool(existing)+"/"+strconv.FormatBool(checkOnly), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + + results := []*RenderResult{{ + Component: "curl", OutputDir: testRenderOutputDir, Status: renderStatusError, lockless: true, + }} + + writeFailureMarkers(env.TestFS, results, true, checkOnly) + assertOriginalRenderOutput(t, env.TestFS, existing) + assert.Equal(t, checkOnly, results[0].Changed) + }) + } + } +} + +func TestPrepareOneComponent_LocklessFailurePreservesOutput(t *testing.T) { + original := newSourcePreparer + newSourcePreparer = func( + _ sourceproviders.SourceManager, _ opctx.FS, _ opctx.EventListener, _ opctx.DryRunnable, + _ ...sources.PreparerOption, + ) (sources.SourcePreparer, error) { + return nil, errors.New("injected source preparation failure") + } + + t.Cleanup(func() { newSourcePreparer = original }) + + for _, existing := range []bool{false, true} { + t.Run(strconv.FormatBool(existing), func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, existing) + comp := components_testutils.NewMockComponent(gomock.NewController(t)) + comp.EXPECT().GetName().AnyTimes().Return("curl") + comp.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Spec: projectconfig.SpecSource{SourceType: projectconfig.SpecSourceTypeUpstream}, + }) + + prep := prepareOneComponent(env.Env, nil, comp, "/staging", "/project/SPECS", + spec.EditorLegacy, false, true) + require.NotNil(t, prep.result) + assert.Contains(t, prep.result.Error, "injected source preparation failure") + assert.True(t, prep.result.lockless) + writeFailureMarkers(env.TestFS, []*RenderResult{prep.result}, true, false) + assertOriginalRenderOutput(t, env.TestFS, existing) + }) + } +} + +type renderOutputFaultFS struct { + afero.Fs + failure string +} + +func (fs *renderOutputFaultFS) Rename(oldName, newName string) error { + if fs.failure == "cross-device" { + return &os.LinkError{Op: "rename", Old: oldName, New: newName, Err: syscall.EXDEV} + } + + if fs.failure == "move" && oldName == "/staging/curl/sidecar" { + return errors.New("injected rename failure") + } + + return fs.Fs.Rename(oldName, newName) +} + +func (fs *renderOutputFaultFS) RemoveAll(path string) error { + if fs.failure == "remove" && path == testRenderOutputDir { + fs.failure = "" + if err := fs.Fs.RemoveAll(filepath.Join(path, "curl.spec")); err != nil { + return err + } + + return errors.New("injected partial removal failure") + } + + return fs.Fs.RemoveAll(path) +} + +func TestLocklessRenderNEVRResolvesAutoreleaseHistory(t *testing.T) { + const processedSpec = "## RPMAUTOSPEC: autorelease, autochangelog\nName: curl\nVersion: 1\nRelease: 7\n" + + for _, testCase := range []struct { + name, existingSpec, committed, rendered string + reject bool + }{ + // rpmspec without history expands '-b 6' to 6, below the processed 7. + {name: "history raises base above processed release", existingSpec: processedSpec, rendered: "1-8"}, + {name: "history release below processed release", existingSpec: processedSpec, rendered: "1-6", reject: true}, + { + name: "unprocessed existing spec uses history", existingSpec: renderTestSpec("1", "%autorelease -b 2", ""), + committed: "1-5", rendered: "1-6", + }, + { + name: "unprocessed existing release above new", existingSpec: renderTestSpec("1", "%autorelease -b 2", ""), + committed: "1-7", rendered: "1-6", reject: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + setupRenderOutput(t, env.TestFS, true) + env.CmdFactory.RegisterCommandInSearchPath("rpmspec") + + for _, root := range []string{testRenderOutputDir, "/before"} { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, "curl.spec"), + []byte(testCase.existingSpec), fileperms.PublicFile)) + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, "/staging/curl/curl.spec", + []byte(renderTestSpec("1", "%autorelease -b 6", "")), fileperms.PublicFile)) + + history := &fakeRenderNEVRHistory{rendered: testSpecInfo(t, testCase.rendered)} + if testCase.committed != "" { + history.committed = testSpecInfo(t, testCase.committed) + } + + stubRenderNEVRHistory(t, history) + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Equal(t, processedSpec, testCase.existingSpec, "only processed specs use plain rpmspec") + assert.Equal(t, testRenderOutputDir, cmd.Dir) + + release := "7" + if cmd.Dir == testRenderStagingDir { + release = "6" + } + + return "name=curl\nepoch=0\nversion=1\nrelease=" + release + "\n", nil + } + + err := moveRenderedOutput(env.Env, testRenderStagingDir, &preparedComponent{ + lockless: true, specFilename: "curl.spec", compOutputDir: testRenderOutputDir, + }, true) + if testCase.reject { + require.ErrorIs(t, err, errRenderNEVRValidation) + assertOriginalRenderOutput(t, env.TestFS, true) + } else { + require.NoError(t, err) + } + + assert.True(t, history.removed) + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_release_carryover.go b/internal/app/azldev/cmds/component/render_release_carryover.go new file mode 100644 index 000000000..a5b59b764 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_release_carryover.go @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +var ( + baseReleaseMacro = regexp.MustCompile(`%(?:\{\??baserelease\}|baserelease\b)`) + baseReleaseLine = regexp.MustCompile(`^[ \t]*%(?:global|define)[ \t]+baserelease[ \t]+\S`) + autoChangelogMacro = regexp.MustCompile(`%(?:\{\??autochangelog\}|autochangelog\b)`) + autoreleaseNoDist = regexp.MustCompile(`(?:^|\s)-n(?:\s|\}|$)`) +) + +func staticRenderHistory( + env *azldev.Env, autospec renderAutorelease, dir, name, release string, history lastRender, editor spec.EditorMode, +) (resolved lastRender, err error) { + if history.spec == "" || env.DryRun() || sources.ReleaseUsesAutorelease(release) { + return history, nil + } + + previous, err := spec.OpenSpec( + strings.NewReader(history.spec), spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) + if err != nil { + return history, fmt.Errorf("parsing committed release spec:\n%w", err) + } + + previousReleases, err := committedReleases(previous) + if err != nil { + return history, err + } + + if !sources.ReleaseUsesAutorelease(previousReleases[0]) { + return history, nil + } + + scratch, err := fileutils.MkdirTemp(env.FS(), filepath.Dir(dir), "azldev-render-static-") + if err != nil { + return history, fmt.Errorf("creating committed autorelease scratch directory:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(env.FS(), scratch, &err) + + snapshot := filepath.Join(scratch, "dist-git") + if _, err := CreateComponentDistGitRepo(env, &DistGitRepoOptions{ + Component: name, OutputDir: snapshot, + }); err != nil { + return history, fmt.Errorf("extracting committed autorelease history:\n%w", err) + } + + filename := name + ".spec" + + release, err = autospec.CalculateRelease(env, snapshot, filename) + if err != nil { + return history, fmt.Errorf("calculating committed autorelease:\n%w", err) + } + + fields := strings.Fields(release) + if len(fields) == 0 { + return history, errors.New("calculating committed autorelease: empty release") + } + + release = fields[len(fields)-1] + if !autoreleaseNoDist.MatchString(previousReleases[0]) { + release += "%{?dist}" + } + + if err := previous.UpdateExistingTag("", "Release", release); err != nil { + return history, fmt.Errorf("carrying calculated autorelease:\n%w", err) + } + + var content strings.Builder + if err := previous.Serialize(&content); err != nil { + return history, fmt.Errorf("serializing committed release spec:\n%w", err) + } + + resolved = history + resolved.spec, err = staticRenderChangelog(env, autospec, snapshot, filename, content.String(), editor) + + return resolved, err +} + +func staticRenderChangelog( + env *azldev.Env, autospec renderAutorelease, dir, filename, content string, editor spec.EditorMode, +) (string, error) { + if !autoChangelogMacro.MatchString(content) { + return content, nil + } + + changelog, err := autospec.GenerateChangelog(env, dir, filename) + if err != nil { + return "", fmt.Errorf("generating committed autorelease changelog:\n%w", err) + } + + start, end, err := renderChangelogBounds([]byte(content), editor) + if err != nil { + return "", err + } + + return content[:start] + "%changelog\n" + changelog + content[end:], nil +} + +func carryStaticRenderRelease( + env *azldev.Env, working *spec.Spec, release string, + history lastRender, editor spec.EditorMode, +) error { + if history.spec == "" || sources.ReleaseUsesAutorelease(release) || env.DryRun() { + return nil + } + + previous, err := spec.OpenSpec(strings.NewReader(history.spec), spec.WithEditor(editor)) + if err != nil { + return fmt.Errorf("parsing committed release spec:\n%w", err) + } + + oldReleases, err := previous.GetTags("", "Release") + if err != nil { + return fmt.Errorf("reading committed Release:\n%w", err) + } + + if !baseReleaseMacro.MatchString(oldReleases[len(oldReleases)-1]) { + return carryCommittedReleases(working, oldReleases) + } + + _, definition, err := baseReleaseDefinition(previous) + if err != nil { + return fmt.Errorf("reading committed 'baserelease' definition:\n%w", err) + } + + line, _, err := baseReleaseDefinition(working) + if err != nil { + return fmt.Errorf("finding upstream 'baserelease' definition:\n%w", err) + } + + working.ReplaceLine(line, definition) + + return nil +} + +// ErrDifferingAutoreleaseTags is returned when a committed spec has several +// '%autorelease' 'Release' tags with different values. +var ErrDifferingAutoreleaseTags = errors.New("committed '%autorelease' 'Release' tags differ") + +// ErrReleaseTagCountMismatch is returned when the upstream and committed specs +// have different numbers of 'Release' tags. +var ErrReleaseTagCountMismatch = errors.New("upstream and committed 'Release' tag counts differ") + +// committedReleases returns the committed 'Release' values, requiring them all +// to be static or all to be the same '%autorelease' value. +func committedReleases(previous *spec.Spec) ([]string, error) { + releases, err := previous.GetTags("", "Release") + if err != nil { + return nil, fmt.Errorf("reading committed Release:\n%w", err) + } + + if err := checkReleaseTagsConsistent(releases); err != nil { + return nil, fmt.Errorf("reading committed Release:\n%w", err) + } + + if sources.ReleaseUsesAutorelease(releases[0]) { + for _, release := range releases[1:] { + if release != releases[0] { + return nil, fmt.Errorf("calculating committed autorelease:\n%w", ErrDifferingAutoreleaseTags) + } + } + } + + return releases, nil +} + +// carryCommittedReleases copies each committed 'Release' value onto the +// upstream 'Release' tag at the same position. +func carryCommittedReleases(working *spec.Spec, oldReleases []string) error { + releases, err := working.GetTags("", "Release") + if err != nil { + return fmt.Errorf("reading upstream Release:\n%w", err) + } + + if len(releases) != len(oldReleases) { + return fmt.Errorf("carrying committed Release: upstream spec has %d 'Release' tags, committed spec has %d:\n%w", + len(releases), len(oldReleases), ErrReleaseTagCountMismatch) + } + + index := 0 + + err = working.UpdateExistingTagFunc("", "Release", func(string) (string, error) { + release := oldReleases[index] + index++ + + return release, nil + }) + if err != nil { + return fmt.Errorf("carrying committed Release:\n%w", err) + } + + return nil +} + +func baseReleaseDefinition(opened *spec.Spec) (line int, definition string, err error) { + var matches int + + err = opened.Visit(func(ctx *spec.Context) error { + if ctx.CurrentSection.SectName == "" && ctx.RawLine != nil && baseReleaseLine.MatchString(*ctx.RawLine) { + line, definition = ctx.CurrentLineNum, *ctx.RawLine + matches++ + } + + return nil + }) + if err != nil { + return 0, "", fmt.Errorf("finding 'baserelease' macro:\n%w", err) + } + + if matches != 1 { + return 0, "", fmt.Errorf("expected one 'baserelease' definition, found %d", matches) + } + + return line, definition, nil +} diff --git a/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go new file mode 100644 index 000000000..b4a0b8510 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_release_carryover_internal_test.go @@ -0,0 +1,324 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareLocklessSpec_AutoreleaseToStatic(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + for _, testCase := range []struct { + name, version, baseline, previous string + fail bool + }{ + {name: "same version", version: "1", baseline: "12%{?dist}"}, + {name: "new version", version: "2", baseline: "1%{?dist}"}, + {name: "without dist", version: "1", baseline: "12", previous: "%{autorelease -b 8 -n}"}, + {name: "release query fails", version: "1", fail: true}, + } { + t.Run(string(editor)+"/"+testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.Project.RenderedSpecsDir = "/project/specs" + env.Config.Components["curl"] = projectconfig.ComponentConfig{Name: "curl"} + env.CmdFactory.RegisterCommandInSearchPath("git") + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + + previous := testCase.previous + if previous == "" { + previous = "%autorelease -b 8" + } + + history := lastRender{spec: renderTestSpec("1", previous, "%autochangelog\n")} + + const path = "/working/curl.spec" + require.NoError(t, fileutils.WriteFile(env.TestFS, path, + []byte(renderTestSpec(testCase.version, "1%{?dist}", "* Upstream entry\n")), + fileperms.PublicFile)) + + var snapshot string + + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + switch cmd.Args[3] { + case renderTestRevParse: + if cmd.Args[4] == "--git-common-dir" { + return "/project/.git", nil + } + + return renderTestProjectRoot, nil + case "clone": + snapshot = cmd.Args[len(cmd.Args)-1] + require.NoError(t, fileutils.MkdirAll(env.TestFS, filepath.Join(snapshot, ".git"))) + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(snapshot, "curl.spec"), + []byte(history.spec), fileperms.PublicFile)) + } + + return "", nil + } + queryErr := errors.New("release query failed") + autospec := &renderAutospecStub{expand: func(dir, filename string) (string, error) { + assert.Equal(t, snapshot, dir) + assert.Equal(t, "curl.spec", filename) + content, err := fileutils.ReadFile(env.TestFS, filepath.Join(dir, filename)) + require.NoError(t, err) + assert.Equal(t, history.spec, string(content)) + + if testCase.fail { + return "", queryErr + } + + return "Calculated release number: 12\n", nil + }} + + err := prepareLocklessSpec(env.Env, autospec, "/working", "curl", editor, history) + if testCase.fail { + require.ErrorIs(t, err, queryErr) + } else { + require.NoError(t, err) + + content, readErr := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, readErr) + assert.Contains(t, string(content), "Release: "+testCase.baseline+"\n") + assert.Contains(t, string(content), "* Full upstream history\n") + assert.NotContains(t, string(content), "%autorelease") + assert.NotContains(t, string(content), "%autochangelog") + + bumped := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + bumped = true + + assert.Equal(t, "rpmdev-bumpspec", cmd.Args[0]) + + return nil + } + _, err = bumpLocklessSpec(env.Env, path, &locklessRenderUpdate{subject: "test transition"}) + require.NoError(t, err) + assert.True(t, bumped) + } + + exists, checkErr := fileutils.Exists(env.TestFS, snapshot) + require.NoError(t, checkErr) + assert.False(t, exists, "remove the temporary history on success and failure") + }) + } + } +} + +func TestPrepareLocklessSpec_CarriesBaseRelease(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, testCase := range []struct { + name, version, definition, release, wantBase, wantBumped string + changed bool + initial bool + }{ + { + name: "same upstream", version: "1", definition: "%global baserelease 18", + release: "%{baserelease}%{?dist}", wantBase: "18", wantBumped: "19", + }, + { + name: "new upstream same version", version: "1", changed: true, + definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", + wantBase: "18", wantBumped: "19", + }, + { + name: "new version", version: "2", changed: true, + definition: "%global baserelease 18", release: "%{baserelease}%{?dist}", + wantBase: "17", wantBumped: "18", + }, + { + name: "define and optional macro", version: "1", definition: "%define baserelease 18", + release: "%{?baserelease}%{?dist}", wantBase: "18", wantBumped: "19", + }, + { + name: "unbraced macro", version: "1", definition: "%global baserelease 18", + release: "%baserelease%{?dist}", wantBase: "18", wantBumped: "19", + }, + { + name: "initial render", version: "1", initial: true, + release: "%{baserelease}%{?dist}", wantBase: "17", wantBumped: "18", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const ( + path = "/working/curl.spec" + oldChangelog = "* Old downstream entry\n" + newRelease = "%{baserelease}.upstream%{?dist}" + releaseTag = "Release:\t " + ) + + oldSpec := testCase.definition + "\n" + + renderTestSpec("1", testCase.release, oldChangelog) + + history := lastRender{spec: oldSpec, previous: "old", current: "old"} + wantChangelog := oldChangelog + + if testCase.initial { + oldSpec = "" + history = lastRender{} + wantChangelog = "* Upstream entry\n" + } + + if testCase.changed { + history.current = "new" + } + + // Upstream context and Release differ; only the macro definition is carried over. + upstream := "# New upstream header\n%define baserelease 17\n" + + renderTestSpec(testCase.version, newRelease, "* Upstream entry\n") + upstream = strings.Replace(upstream, "Release: ", releaseTag, 1) + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(upstream), fileperms.PublicFile)) + env.CmdFactory.RegisterCommandInSearchPath("rpmdev-bumpspec") + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "abc123 Upstream change\n", nil + } + + var calls int + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + calls++ + + assert.Equal(t, path, cmd.Args[len(cmd.Args)-1]) + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + + definition := testCase.definition + if testCase.wantBase == "17" { + definition = "%define baserelease 17" + } + + assert.Contains(t, string(content), definition+"\n") + assert.Contains(t, string(content), releaseTag+newRelease+"\n") + assert.Contains(t, string(content), "%changelog\n"+wantChangelog) + + return fileutils.WriteFile(env.TestFS, path, + []byte(strings.Replace(string(content), "baserelease "+testCase.wantBase, + "baserelease "+testCase.wantBumped, 1)), fileperms.PublicFile) + } + update := &locklessRenderUpdate{subject: "updated curl"} + require.NoError(t, collectRenderUpstreamChanges(env.Env, "/working", "curl", + testCase.version, history, update)) + err := prepareLocklessSpec(env.Env, &renderAutospecStub{}, "/working", "curl", editor, history) + require.NoError(t, err) + assert.Zero(t, calls) + + _, err = bumpLocklessSpec(env.Env, path, update) + require.NoError(t, err) + assert.Equal(t, 1, calls) + + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "baserelease "+testCase.wantBumped+"\n") + assert.Contains(t, string(content), releaseTag+newRelease+"\n") + assert.Equal(t, oldSpec, history.spec) + }) + } + }) + } +} + +func TestPrepareLocklessSpec_PreservesAutoreleaseSpacing(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, spacing := range []string{" ", "\t \t", ""} { + t.Run(spacing, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + const path = "/working/curl.spec" + + upstream := strings.Replace(renderTestSpec("1", "%autorelease -b 2 -e test", "%autochangelog\n"), + "Release: ", "Release:"+spacing, 1) + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(upstream), fileperms.PublicFile)) + + err := prepareLocklessSpec(env.Env, &renderAutospecStub{base: "9"}, + "/working", "curl", editor, lastRender{}) + require.NoError(t, err) + content, err := fileutils.ReadFile(env.TestFS, path) + require.NoError(t, err) + assert.Contains(t, string(content), "Release:"+spacing+"%autorelease -b 9 -e test\n") + }) + } + }) + } +} + +func TestCarryStaticRenderRelease(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, testCase := range []struct { + name, previous, upstream, oldRelease, wantRelease, wantError string + }{ + { + name: "literal release with unrelated baserelease definition", + previous: "%global baserelease 18\n", upstream: "%global baserelease 17\n", + oldRelease: "9%{?dist}", wantRelease: "9%{?dist}", + }, + { + name: "similarly named macro", oldRelease: "%{baserelease_extra}", + wantRelease: "%{baserelease_extra}", + }, + { + name: "missing committed definition", upstream: "%global baserelease 17\n", + oldRelease: "%{baserelease}", wantError: "reading committed 'baserelease' definition", + }, + { + name: "missing upstream definition", previous: "%global baserelease 18\n", + oldRelease: "%{baserelease}", wantError: "finding upstream 'baserelease' definition", + }, + { + name: "ambiguous committed definition", + previous: "%global baserelease 18\n%define baserelease 19\n", + upstream: "%global baserelease 17\n", oldRelease: "%{baserelease}", + wantError: "expected one 'baserelease' definition, found 2", + }, + { + name: "ambiguous upstream definition", previous: "%global baserelease 18\n", + upstream: "%global baserelease 16\n%define baserelease 17\n", + oldRelease: "%{baserelease}", wantError: "expected one 'baserelease' definition, found 2", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + upstream := testCase.upstream + renderTestSpec("1", "1%{?dist}", "") + opened, err := spec.OpenSpec(strings.NewReader(upstream), spec.WithEditor(editor)) + require.NoError(t, err) + + history := lastRender{spec: testCase.previous + renderTestSpec("1", testCase.oldRelease, "")} + + err = carryStaticRenderRelease(env.Env, opened, "1%{?dist}", history, editor) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + + return + } + + require.NoError(t, err) + release, err := opened.GetLastTag("", "Release") + require.NoError(t, err) + assert.Equal(t, testCase.wantRelease, release) + + if testCase.upstream != "" { + _, definition, err := baseReleaseDefinition(opened) + require.NoError(t, err) + assert.Equal(t, strings.TrimSuffix(testCase.upstream, "\n"), definition) + } + }) + } + }) + } +} diff --git a/internal/app/azldev/cmds/component/render_test.go b/internal/app/azldev/cmds/component/render_test.go index 00a03def8..0930d293e 100644 --- a/internal/app/azldev/cmds/component/render_test.go +++ b/internal/app/azldev/cmds/component/render_test.go @@ -4,6 +4,8 @@ package component_test import ( + "os/exec" + "slices" "testing" componentcmds "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" @@ -49,6 +51,26 @@ func TestNewRenderCmd_Flags(t *testing.T) { require.NotNil(t, checkOnlyFlag, "check-only flag should be registered") assert.Equal(t, "false", checkOnlyFlag.DefValue) + noChangeRebuildFlag := cmd.Flags().Lookup("no-change-rebuild") + require.NotNil(t, noChangeRebuildFlag) + require.NoError(t, cmd.ParseFlags([]string{"--no-change-rebuild"})) + assert.Equal(t, "true", noChangeRebuildFlag.Value.String()) + + commentFlag := cmd.Flags().Lookup("changelog-comment") + require.NotNil(t, commentFlag) + + noGitCommitFlag := cmd.Flags().Lookup("no-git-commit") + require.NotNil(t, noGitCommitFlag) + require.NoError(t, cmd.ParseFlags([]string{"--changelog-comment", "Fix curl patches", "--no-git-commit"})) + assert.Equal(t, "Fix curl patches", commentFlag.Value.String()) + assert.Equal(t, "true", noGitCommitFlag.Value.String()) + + allowUncommittedFlag := cmd.Flags().Lookup("allow-uncommitted") + require.NotNil(t, allowUncommittedFlag) + assert.Equal(t, "false", allowUncommittedFlag.DefValue) + require.NoError(t, cmd.ParseFlags([]string{"--allow-uncommitted"})) + assert.Equal(t, "true", allowUncommittedFlag.Value.String()) + rpmDevBumpspecFlag := cmd.Flags().Lookup("rpmdev-bumpspec") require.NotNil(t, rpmDevBumpspecFlag) assert.Equal(t, "false", rpmDevBumpspecFlag.DefValue) @@ -61,7 +83,10 @@ func TestRenderCmd_NoComponents(t *testing.T) { testEnv := testutils.NewTestEnv(t) cmd := componentcmds.NewRenderCmd() - cmd.SetArgs([]string{"-o", "SPECS", "nonexistent-component"}) + cmd.SetArgs([]string{ + "-o", "SPECS", "--no-change-rebuild", "--no-git-commit", + "--changelog-comment", "Fix patches", "nonexistent-component", + }) err := cmd.ExecuteContext(testEnv.Env) @@ -70,6 +95,61 @@ func TestRenderCmd_NoComponents(t *testing.T) { assert.Contains(t, err.Error(), "component not found") } +func TestRenderCmd_AllowUncommitted(t *testing.T) { + for _, testCase := range []struct { + name string + allow bool + }{ + {name: "dirty repository refused"}, + {name: "override skips status", allow: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + env.AddUpstreamComponent(t, "curl") + env.Config.LocklessComponents = map[string]bool{"curl": true} + config := env.Config.Components["curl"] + config.Spec.UpstreamCommit = "configured-commit" + env.Config.Components["curl"] = config + queried := false + original := env.CmdFactory.RunAndGetOutputHandler + env.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + if slices.Contains(cmd.Args, "--porcelain=v1") { + queried = true + + return "?? scratch\n", nil + } + + return original(cmd) + } + cmd := componentcmds.NewRenderCmd() + + args := []string{"-p", "curl", "-o", "SPECS", "--fail-on-error"} + if testCase.allow { + args = append(args, "--allow-uncommitted") + } + + cmd.SetArgs(args) + err := cmd.ExecuteContext(env.Env) + + assert.Equal(t, !testCase.allow, queried) + require.Error(t, err) + + if !testCase.allow { + assert.Contains(t, err.Error(), "uncommitted or untracked files") + } else { + assert.NotContains(t, err.Error(), "uncommitted or untracked files") + } + }) + } +} + +func TestRenderCmd_ChangelogCommentRequiresValue(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + cmd := componentcmds.NewRenderCmd() + cmd.SetArgs([]string{"--changelog-comment"}) + require.ErrorContains(t, cmd.ExecuteContext(testEnv.Env), "flag needs an argument") +} + func TestRenderCmd_NoOutputDir(t *testing.T) { testEnv := testutils.NewTestEnv(t) diff --git a/internal/app/azldev/cmds/component/update.go b/internal/app/azldev/cmds/component/update.go index 9da1b4f75..1652f1783 100644 --- a/internal/app/azldev/cmds/component/update.go +++ b/internal/app/azldev/cmds/component/update.go @@ -59,6 +59,10 @@ For local components, this computes a content hash of the spec directory. Subsequent commands (render, build) use the locked state for deterministic, reproducible results. +Components listed in lockless-components.txt beside the top-level azldev.toml +are skipped, including with --bump. Use refresh-upstream-commit for them. +Their existing lock files are left untouched. + If a component fails to resolve, successful components are still written before the command returns an error. Failed components are left unchanged, and orphan pruning is skipped for that run. @@ -100,7 +104,7 @@ Cannot be combined with --bump.`, ValidArgsFunction: components.GenerateComponentNameCompletions, } - addComponentFilterOptions(cmd, &options.ComponentFilter, cmdOptions{}) + addComponentFilterOptions(cmd, &options.ComponentFilter) cmd.Flags().BoolVar(&options.Bump, "bump", false, "increment the manual-rebuild counter to trigger a new release") @@ -161,6 +165,14 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat return nil, fmt.Errorf("%w: --bump and --check-only are mutually exclusive", azldev.ErrInvalidUsage) } + if env.WithoutLockfile() { + return []UpdateResult{}, nil + } + + return updateLockedComponents(env, options) +} + +func updateLockedComponents(env *azldev.Env, options *UpdateComponentOptions) ([]UpdateResult, error) { resolver := components.NewResolver(env) // Suppress staleness warnings — we're about to refresh the locks ourselves, // so warning the user to "run component update" would be self-referential noise. @@ -184,6 +196,8 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat return nil, errors.New("no components matched the filter") } + lockedComps := componentsForLockMode(env, comps, false) + // Resolve upstream commits in parallel (no-op for empty list). store := env.LockStore() if store == nil { @@ -195,7 +209,7 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat // existing locks and should not delete entries for components that may // have been removed. if options.Bump { - results, bumpErr := bumpComponents(env, store, comps, options) + results, bumpErr := bumpComponents(env, store, lockedComps, options) if bumpErr != nil { return filterDisplayResults(results), bumpErr } @@ -205,7 +219,7 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat return filterDisplayResults(results), nil } - results := resolveSourceIdentitiesParallel(env, comps, store) + results := resolveSourceIdentitiesParallel(env, lockedComps, store) // Don't save if the context was cancelled (Ctrl+C). if env.Context().Err() != nil { diff --git a/internal/app/azldev/cmds/component/update_test.go b/internal/app/azldev/cmds/component/update_test.go index fea8742ab..dabcb759e 100644 --- a/internal/app/azldev/cmds/component/update_test.go +++ b/internal/app/azldev/cmds/component/update_test.go @@ -22,6 +22,41 @@ import ( const testLockDir = "/project/locks" +func TestUpdateComponents_SkipsLocklessAndPreservesTheirLocks(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + setupMockGit(env, "resolved-commit") + addUpstreamComponent(env, "curl") + addUpstreamComponent(env, "rust-asyncband") + + ignoredLock := []byte("corrupt unused lock") + require.NoError(t, fileutils.WriteFile(env.TestFS, testLockDir+"/rust-asyncband.lock", + ignoredLock, fileperms.PublicFile)) + + results, err := componentcmds.UpdateComponents(env.Env, &componentcmds.UpdateComponentOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "curl", results[0].Component) + + lock, err := env.Env.LockStore().Get("curl") + require.NoError(t, err) + assert.Equal(t, "resolved-commit", lock.UpstreamCommit) + + results, err = componentcmds.UpdateComponents(env.Env, &componentcmds.UpdateComponentOptions{ + ComponentFilter: components.ComponentFilter{IncludeAllComponents: true}, + Bump: true, + }) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "curl", results[0].Component) + + unchanged, err := fileutils.ReadFile(env.TestFS, testLockDir+"/rust-asyncband.lock") + require.NoError(t, err) + assert.Equal(t, ignoredLock, unchanged) +} + func TestNewUpdateCmd(t *testing.T) { cmd := componentcmds.NewUpdateCmd() require.NotNil(t, cmd) diff --git a/internal/app/azldev/cmds/docs/agent.go b/internal/app/azldev/cmds/docs/agent.go index 0591e1e5f..bb65a46f7 100644 --- a/internal/app/azldev/cmds/docs/agent.go +++ b/internal/app/azldev/cmds/docs/agent.go @@ -35,8 +35,8 @@ type InstalledAgentFile struct { } // Called once when the app is initialized; registers the 'agent' command tree under 'docs'. -func agentOnAppInit(app *azldev.App, parentCmd *cobra.Command) { - parentCmd.AddCommand(newAgentCmd(agentskill.NewCatalog(app.WithoutLockfile()))) +func agentOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(newAgentCmd(agentskill.NewCatalog(false))) } func newAgentCmd(catalog agentskill.Catalog) *cobra.Command { @@ -89,8 +89,8 @@ MCP server. Directory paths in the emitted content (such as the lock, generated upstream-commit, and rendered-spec directories) are resolved from the loaded azldev.toml, falling back to azldev's built-in defaults when no configuration is -found. The emitted content also reflects the mode azldev runs in, so pass ---without-lockfile to describe the lock-file-free workflow. The bindings reflect +found. By default, the content includes both lock-file and lockless workflows +selected per component by lockless-components.txt. The bindings reflect the project azldev runs in, so pair --output-dir with -C pointing at the target repository when scaffolding a different repo.`, Example: ` # Write agent files into the current repository @@ -151,7 +151,7 @@ skill to list the available skills.`, return fmt.Errorf("failed to get command environment:\n%w", err) } - name, list, err := resolveShowSkill(agentSkillCatalog(env), skillName) + name, list, err := resolveShowSkill(agentskill.NewCatalog(false), skillName) if err != nil { return err } @@ -165,7 +165,7 @@ skill to list the available skills.`, return nil } - doc, err := agentSkillCatalog(env).SkillDocument(name, agentSkillParams(env, cmd.Root())) + doc, err := agentskill.NewCatalog(false).SkillDocument(name, agentSkillParams(env, cmd.Root())) if err != nil { return fmt.Errorf("failed to render azldev skill:\n%w", err) } @@ -193,7 +193,7 @@ func InstallAgentFiles( return nil, err } - files, err := agentSkillCatalog(env).Files(layout, agentSkillParams(env, rootCmd), options.Full) + files, err := agentskill.NewCatalog(false).Files(layout, agentSkillParams(env, rootCmd), options.Full) if err != nil { return nil, fmt.Errorf("failed to render azldev agent files:\n%w", err) } @@ -320,12 +320,6 @@ func emitMCPConfig( return InstalledAgentFile{Path: destPath, Written: true}, nil } -// agentSkillCatalog returns the skill catalog describing the mode azldev is running -// in, so the emitted and served content matches the commands the user actually has. -func agentSkillCatalog(env *azldev.Env) agentskill.Catalog { - return agentskill.NewCatalog(env != nil && env.WithoutLockfile()) -} - // agentSkillParams gathers the dynamic values injected into the emitted and served agent content, // including the target-repo bindings resolved from the loaded project configuration. func agentSkillParams(env *azldev.Env, rootCmd *cobra.Command) agentskill.Params { diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder.go b/internal/app/azldev/core/componentbuilder/componentbuilder.go index 8cb376195..956719d9e 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder.go @@ -46,6 +46,7 @@ type Builder struct { sourcePreparer sources.SourcePreparer buildEnv buildenv.RPMAwareBuildEnv workDirFactory *workdir.Factory + distGitDir string } // Ensure that [Builder] implements the [ComponentBuilder] interface. @@ -80,6 +81,11 @@ func (b *Builder) BuildSourcePackage( return packagePath, nil } +// UseDistGitSources builds the prepared repository without replaying overlays or synthetic history. +func (b *Builder) UseDistGitSources(dir string) { + b.distGitDir = dir +} + func (b *Builder) BuildBinaryPackage( ctx context.Context, component components.Component, sourcePackagePath string, localRepoPaths []string, @@ -114,6 +120,10 @@ func (b *Builder) buildSRPM( func (b *Builder) prepSourcesForSRPM( ctx context.Context, component components.Component, ) (preparedSourcesDir string, err error) { + if b.distGitDir != "" { + return b.distGitDir, nil + } + // Prep sources. prepEvent := b.eventListener.StartEvent("Acquiring and preparing sources", "component", component.GetName()) @@ -162,6 +172,9 @@ func (b *Builder) buildSRPMFromPreparedSources( LocalRepoPaths: localRepoPaths, }, } + if b.distGitDir != "" { + srpmOptions.RPMAutospec = true + } err = buildEnv.BuildSRPM(ctx, preparedSpecPath, preparedSourcesDir, srpmOutputDir, srpmOptions) if err != nil { diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go index 7fdf7ec88..4235e4dc7 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go @@ -159,6 +159,44 @@ func TestBuilderNew(t *testing.T) { require.NotNil(t, params) } +func TestBuildFromDistGit(t *testing.T) { + params := setupBuilder(t) + comp := params.createTestComponent(t) + + const dir = "/committed-dist-git" + + // A nil preparer ensures dist-git builds cannot accidentally replay the old preparation pipeline. + builder := componentbuilder.New(params.testEnv.Env, params.testEnv.FS(), params.testEnv.Env, + nil, params.buildEnv, params.workDirFactory) + builder.UseDistGitSources(dir) + params.buildEnv.EXPECT().BuildSRPM(gomock.Any(), dir+"/"+comp.GetName()+".spec", dir, + gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _, _, output string, options buildenv.SRPMBuildOptions) error { + require.True(t, options.RPMAutospec) + require.Empty(t, options.With) + require.Empty(t, options.Without) + require.Empty(t, options.Defines) + require.Equal(t, []string{"/local-repo"}, options.LocalRepoPaths) + + return fileutils.WriteFile(params.testEnv.FS(), filepath.Join(output, "package.src.rpm"), + []byte("srpm"), fileperms.PublicFile) + }) + srpm, err := builder.BuildSourcePackage(params.testEnv.Env, comp, []string{"/local-repo"}, "/output") + require.NoError(t, err) + params.buildEnv.EXPECT().BuildRPM(gomock.Any(), srpm, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ string, _ string, options buildenv.RPMBuildOptions) error { + require.True(t, options.NoCheck) + require.Empty(t, options.With) + require.Empty(t, options.Without) + require.Empty(t, options.Defines) + require.Equal(t, []string{"/local-repo"}, options.LocalRepoPaths) + + return nil + }) + _, err = builder.BuildBinaryPackage(params.testEnv.Env, comp, srpm, []string{"/local-repo"}, "/output", true) + require.NoError(t, err) +} + func TestBuildSourcePackage(t *testing.T) { const testOutputDir = "/output" diff --git a/internal/app/azldev/core/components/resolver.go b/internal/app/azldev/core/components/resolver.go index fad536a40..e4f0b2986 100644 --- a/internal/app/azldev/core/components/resolver.go +++ b/internal/app/azldev/core/components/resolver.go @@ -581,6 +581,12 @@ func (r *Resolver) createComponentFromConfig(componentConfig *projectconfig.Comp // goes into the separate Locked field, preserving the manifest/lock boundary: // Spec.UpstreamCommit = user intent, Locked.UpstreamCommit = resolved reality. func (r *Resolver) populateFromLock(config *projectconfig.ComponentConfig) { + if r.env.ComponentWithoutLockfile(config.Name) { + config.Locked = nil + + return + } + reader := r.env.LockReader() if reader == nil { return @@ -912,11 +918,17 @@ func (r *Resolver) validateLockFiles(resolved *ComponentSet, checkOrphans bool, // Build resolved config map from the component set. resolvedConfigs := make(map[string]projectconfig.ComponentConfig, resolved.Len()) + + var ignoredComponents []string + for _, comp := range resolved.Components() { resolvedConfigs[comp.GetName()] = *comp.GetConfig() + if r.env.ComponentWithoutLockfile(comp.GetName()) { + ignoredComponents = append(ignoredComponents, comp.GetName()) + } } - stale, orphans, err := reader.ValidateConsistency(resolvedConfigs, checkOrphans) + stale, orphans, err := reader.ValidateConsistency(resolvedConfigs, checkOrphans, ignoredComponents...) if err == nil { return nil } diff --git a/internal/app/azldev/core/sources/bumpspec.go b/internal/app/azldev/core/sources/bumpspec.go index ee1a54bf5..5f4da98ad 100644 --- a/internal/app/azldev/core/sources/bumpspec.go +++ b/internal/app/azldev/core/sources/bumpspec.go @@ -206,7 +206,7 @@ func prepareRPMDevBumpspecHome(ctx opctx.Ctx, request RPMDevBumpspecRequest) err return fmt.Errorf("failed to create isolated RPM wrapper directory %#q:\n%w", filepath.Dir(rpmWrapperPath), err) } - if err := fileutils.WriteFile(ctx.FS(), rpmWrapperPath, []byte(rpmdevBumpspecWrapper(request)), + if err := fileutils.WriteFile(ctx.FS(), rpmWrapperPath, []byte(GenerateRPMDevBumpspecWrapper(request)), fileperms.PublicExecutable); err != nil { return fmt.Errorf("failed to create isolated RPM wrapper %#q:\n%w", rpmWrapperPath, err) } @@ -214,7 +214,9 @@ func prepareRPMDevBumpspecHome(ctx opctx.Ctx, request RPMDevBumpspecRequest) err return nil } -func rpmdevBumpspecWrapper(request RPMDevBumpspecRequest) string { +// GenerateRPMDevBumpspecWrapper generates an RPM wrapper with request-local spec +// and build context. RPMDEV_BUMPSPEC_ORIGINAL_PATH must contain the original PATH. +func GenerateRPMDevBumpspecWrapper(request RPMDevBumpspecRequest) string { args := append([]string{"rpm"}, rpmdevBumpspecContextArgs(request)...) quoted := make([]string, len(args)) diff --git a/internal/app/azldev/core/sources/mockprocessor.go b/internal/app/azldev/core/sources/mockprocessor.go index a10ff7107..9fb2c0a72 100644 --- a/internal/app/azldev/core/sources/mockprocessor.go +++ b/internal/app/azldev/core/sources/mockprocessor.go @@ -87,6 +87,7 @@ func NewMockProcessor(ctx opctx.Ctx, mockConfigPath string, opts ...MockProcesso type ComponentInput struct { Name string // Component name (matches subdirectory in staging dir) SpecFilename string // Just the filename, e.g., "curl.spec" + SkipSpectool bool } // ComponentMockResult holds the mock processing result for one component. @@ -402,6 +403,7 @@ func (p *MockProcessor) CalculateRelease(ctx context.Context, specHostDir, specF type componentInputJSON struct { Name string `json:"name"` SpecFilename string `json:"specFilename"` + SkipSpectool bool `json:"skipSpectool,omitempty"` } // componentResultJSON mirrors the JSON output from render_process.py. diff --git a/internal/app/azldev/core/sources/release.go b/internal/app/azldev/core/sources/release.go index 6cb225af8..c59faeaaa 100644 --- a/internal/app/azldev/core/sources/release.go +++ b/internal/app/azldev/core/sources/release.go @@ -208,7 +208,7 @@ func (p *sourcePreparerImpl) readAndBumpRelease( Value: newRelease, } - if err := ApplySpecOverlayToFileInPlace(p.fs, overlay, specPath, spec.WithEditor(p.specEditor)); err != nil { + if err := ApplySpecOverlayToFileInPlace(p.fs, overlay, specPath, p.specOptions()...); err != nil { return fmt.Errorf("failed to apply release bump overlay for component %#q:\n%w", component.GetName(), err) } diff --git a/internal/app/azldev/core/sources/release_internal_test.go b/internal/app/azldev/core/sources/release_internal_test.go index 629fc5ef6..86cdfd912 100644 --- a/internal/app/azldev/core/sources/release_internal_test.go +++ b/internal/app/azldev/core/sources/release_internal_test.go @@ -211,6 +211,53 @@ func TestTryBumpStaticRelease_AutoreleaseSkips(t *testing.T) { require.NoError(t, preparer.tryBumpStaticRelease(comp, filepath.Join(testSourcesDir, "test-pkg"), 3)) } +func TestSourcePreparer_ReleaseFormattingByLockMode(t *testing.T) { + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + for _, mode := range []struct { + name string + lockless bool + preserveExplicit bool + }{ + {name: "locked"}, + {name: "lockless", lockless: true}, + {name: "explicit", preserveExplicit: true}, + } { + t.Run(fmt.Sprintf("%s/%s", editor, mode.name), func(t *testing.T) { + ctx := newBumpspecCtx() + + preparer := &sourcePreparerImpl{ + fs: ctx.FS(), dryRunnable: ctx, specEditor: editor, withoutLockfile: mode.lockless, + } + if mode.preserveExplicit { + WithPreservedReleaseFormatting()(preparer) + } + + specPath := writeTestSpec(t, ctx.FS(), "curl", "Name: curl\nVersion: 1\nrelease:\t 1%{?dist}\n") + overlay := projectconfig.ComponentOverlay{ + Type: projectconfig.ComponentOverlayUpdateSpecTag, Tag: "Release", Value: "2%{?dist}", + } + require.NoError(t, preparer.applyOverlayList([]projectconfig.ComponentOverlay{overlay}, + filepath.Dir(specPath), specPath)) + + prefix := "Release: " + if mode.lockless || mode.preserveExplicit { + prefix = "release:\t " + } + + content, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.Contains(t, string(content), "\n"+prefix+"2%{?dist}\n") + + comp := mockReleaseComponent(gomock.NewController(t), "curl", projectconfig.ReleaseCalculationStatic) + require.NoError(t, preparer.tryBumpStaticRelease(comp, filepath.Dir(specPath), 1)) + content, err = fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.Contains(t, string(content), "\n"+prefix+"3%{?dist}\n") + }) + } + } +} + func TestTryBumpStaticRelease_StaticBumps(t *testing.T) { ctrl := gomock.NewController(t) memFS := afero.NewMemMapFs() diff --git a/internal/app/azldev/core/sources/render_process.py b/internal/app/azldev/core/sources/render_process.py index 041a54f61..f8d7f58d1 100755 --- a/internal/app/azldev/core/sources/render_process.py +++ b/internal/app/azldev/core/sources/render_process.py @@ -40,7 +40,9 @@ EXPECTED_ARG_COUNT = 3 -def process_component(staging_dir: str, name: str, spec_filename: str) -> dict[str, str | None]: +def process_component( + staging_dir: str, name: str, spec_filename: str, *, skip_spectool: bool = False +) -> dict[str, str | None]: """Run rpmautospec + spectool for a single component, returning a result dict. Trust boundary: name and spec_filename are validated by BatchProcess in @@ -65,6 +67,9 @@ def process_component(staging_dir: str, name: str, spec_filename: str) -> dict[s "error": f"rpmautospec failed: {rpa_result.stderr.strip()}", } + if skip_spectool: + return {"name": name, "specFiles": "", "error": None} + # spectool: list source and patch files. st_result = subprocess.run( [ @@ -120,6 +125,7 @@ def main() -> int: staging_dir, comp["name"], comp["specFilename"], + skip_spectool=comp.get("skipSpectool", False), ): comp["name"] for comp in inputs } diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 4edeb30b6..4cdeff6f9 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -114,7 +114,7 @@ func WithDirtyDetection() PreparerOption { // WithoutLockfileHistory returns a [PreparerOption] that derives synthetic history // from the component's generated upstream-commit TOML instead of its lock file. -// It selects the behavior of the global '--without-lockfile' flag; the lock reader +// It selects lockless component behavior; the lock reader // and release version passed to [WithGitRepo] are unused in that mode. func WithoutLockfileHistory() PreparerOption { return func(p *sourcePreparerImpl) { @@ -122,11 +122,26 @@ func WithoutLockfileHistory() PreparerOption { } } +// WithoutSyntheticHistory preserves the upstream Git repository without generating +// synthetic commits or applying their release adjustments. +func WithoutSyntheticHistory() PreparerOption { + return func(p *sourcePreparerImpl) { + p.skipSyntheticHistory = true + } +} + // WithSpecEditor selects the [spec.EditorMode] used for source preparation. func WithSpecEditor(mode spec.EditorMode) PreparerOption { return func(p *sourcePreparerImpl) { p.specEditor = mode } } +// WithPreservedReleaseFormatting returns a [PreparerOption] that keeps existing +// 'Release' tag casing and spacing when overlays edit its value, for lockless +// callers that do not otherwise select [WithoutLockfileHistory]. +func WithPreservedReleaseFormatting() PreparerOption { + return func(p *sourcePreparerImpl) { p.preserveReleaseFormatting = true } +} + // WithSkipLookaside returns a [PreparerOption] that skips all lookaside cache // downloads during source preparation. This includes both explicit source file // downloads ([SourceManager.FetchFiles]) and lookaside extraction during @@ -140,6 +155,19 @@ func WithSkipLookaside() PreparerOption { } } +// WithSkipLocalSpecConfig excludes the TOML defining the local spec path from fetched files. +func WithSkipLocalSpecConfig() PreparerOption { + return func(p *sourcePreparerImpl) { p.skipLocalSpecConfig = true } +} + +// WithFetchedSources runs prepare on the fetched checkout before overlays and +// synthetic history are applied. +func WithFetchedSources(prepare func(context.Context, components.Component, string) error) PreparerOption { + return func(p *sourcePreparerImpl) { + p.prepareFetchedSources = prepare + } +} + // WithUpstreamProvenance returns a [PreparerOption] that enables emission of // the %fedora_upstream_version and %fedora_upstream_release macros for Fedora // upstream components. distTag is the resolved %{?dist} expansion (e.g. @@ -153,9 +181,9 @@ func WithUpstreamProvenance(distTag string) PreparerOption { // WithMockProcessor returns a [PreparerOption] that supplies the mock chroot // used to resolve an rpmautospec %autorelease when emitting -// %fedora_upstream_release. Without it, upstream components whose Release uses -// %autorelease get no release macro (best-effort). Running rpmautospec inside -// mock keeps it out of azldev's host dependencies. A nil processor is ignored. +// %fedora_upstream_release. Without a resolver, upstream components whose Release +// uses %autorelease get no release macro (best-effort). Lockless rendering uses +// [WithAutoreleaseResolver] instead to run on the host. A nil processor is ignored. func WithMockProcessor(mp *MockProcessor) PreparerOption { return func(p *sourcePreparerImpl) { if mp != nil { @@ -164,6 +192,13 @@ func WithMockProcessor(mp *MockProcessor) PreparerOption { } } +// WithAutoreleaseResolver selects how upstream provenance release numbers are calculated. +func WithAutoreleaseResolver(resolver AutoreleaseResolver) PreparerOption { + return func(p *sourcePreparerImpl) { + p.autoreleaseResolver = resolver + } +} + // WithAllowNoHashes returns a [PreparerOption] that allows source file // references to omit their [projectconfig.SourceFileReference.Hash] value. // When set, any source file that lacks a hash will have its hash computed @@ -184,20 +219,27 @@ func WithAllowNoHashes() PreparerOption { // Standard implementation of the [SourcePreparer] interface. type sourcePreparerImpl struct { - sourceManager sourceproviders.SourceManager - specEditor spec.EditorMode - fs opctx.FS - eventListener opctx.EventListener - dryRunnable opctx.DryRunnable + sourceManager sourceproviders.SourceManager + prepareFetchedSources func(context.Context, components.Component, string) error + specEditor spec.EditorMode + // preserveReleaseFormatting keeps 'Release' tag formatting during overlay + // edits. Set via [WithPreservedReleaseFormatting]. + preserveReleaseFormatting bool + fs opctx.FS + eventListener opctx.EventListener + dryRunnable opctx.DryRunnable // withGitRepo, when true, enables dist-git creation by preserving the // upstream .git directory and generating synthetic commit history. - withGitRepo bool + withGitRepo bool + skipSyntheticHistory bool // skipLookaside, when true, skips all lookaside cache downloads during // source preparation. Git-tracked files are still fetched. skipLookaside bool + skipLocalSpecConfig bool + // cmdFactory is used to shell out to git for fingerprint change detection // in the project repository. Set via [WithGitRepo]. cmdFactory opctx.CmdFactory @@ -229,11 +271,11 @@ type sourcePreparerImpl struct { // [WithUpstreamProvenance]. upstreamDistTag string - // autoreleaseResolver runs rpmautospec inside a mock chroot to resolve an + // autoreleaseResolver runs rpmautospec to resolve an // %autorelease Release into a concrete Fedora release number for // %fedora_upstream_release. Nil disables autorelease resolution (the release - // macro is skipped for such specs). Set via [WithMockProcessor]. - autoreleaseResolver autoreleaseResolver + // macro is skipped for such specs). Set via [WithMockProcessor] or [WithAutoreleaseResolver]. + autoreleaseResolver AutoreleaseResolver releaseStrategy releaseStrategy bumpspecCtx opctx.Ctx @@ -303,25 +345,20 @@ func (p *sourcePreparerImpl) PrepareSources( } } - // Preserve the upstream .git directory only when dist-git creation is - // requested via --with-git. This is required so that overlay commits can be - // appended on top of the upstream commit log during synthetic history generation. - var fetchOpts []sourceproviders.FetchComponentOption - if applyOverlays && p.withGitRepo { - fetchOpts = append(fetchOpts, sourceproviders.WithPreserveGitDir()) - } - - if p.skipLookaside { - fetchOpts = append(fetchOpts, sourceproviders.WithSkipLookaside()) - } - // Fetch the component first (spec, sidecar files, and upstream source tarballs). - err := p.sourceManager.FetchComponent(ctx, component, outputDir, fetchOpts...) + err := p.sourceManager.FetchComponent(ctx, component, outputDir, p.componentFetchOptions(applyOverlays)...) if err != nil { return fmt.Errorf("failed to fetch sources for component %#q:\n%w", component.GetName(), err) } + // Capture provenance before fetched-source preparation changes the upstream spec. + macros := p.resolveMacros(ctx, component, outputDir, applyOverlays) + + if err := p.prepareFetched(ctx, component, outputDir); err != nil { + return fmt.Errorf("preparing fetched sources for %#q:\n%w", component.GetName(), err) + } + // Fetch custom and downloaded source files. if !p.skipLookaside { if err := p.sourceManager.FetchFiles(ctx, component, outputDir); err != nil { @@ -333,7 +370,7 @@ func (p *sourcePreparerImpl) PrepareSources( fingerprintConfig := component.GetConfig() if applyOverlays { - repackedArchives, err := p.applyOverlaysToSources(ctx, component, outputDir) + repackedArchives, err := p.applyOverlaysToSources(component, outputDir, macros) if err != nil { return err } @@ -360,6 +397,37 @@ func (p *sourcePreparerImpl) PrepareSources( return nil } +func (p *sourcePreparerImpl) componentFetchOptions(applyOverlays bool) []sourceproviders.FetchComponentOption { + var options []sourceproviders.FetchComponentOption + if p.withoutLockfile { + options = append(options, sourceproviders.WithSingleBranch()) + } + + if p.withGitRepo && (applyOverlays || p.skipSyntheticHistory) { + options = append(options, sourceproviders.WithPreserveGitDir()) + } + + if p.skipLookaside { + options = append(options, sourceproviders.WithSkipLookaside()) + } + + if p.skipLocalSpecConfig { + options = append(options, sourceproviders.WithSkipLocalSpecConfig()) + } + + return options +} + +func (p *sourcePreparerImpl) prepareFetched( + ctx context.Context, component components.Component, dir string, +) error { + if p.prepareFetchedSources == nil { + return nil + } + + return p.prepareFetchedSources(ctx, component, dir) +} + func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.Component) error { config := component.GetConfig() if err := config.ValidateArchiveOverlays(p.allowNoHashes); err != nil { @@ -369,39 +437,56 @@ func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.C return nil } -// applyOverlaysToSources writes the macros file and then applies all overlays. +// applyOverlaysToSources prepares macro definitions and then applies all overlays. // It returns the names of any archives that were repacked by archive overlays // (empty in dry-run mode or when no archive overlays ran), so the caller can // rehash exactly those entries in the 'sources' file. func (p *sourcePreparerImpl) applyOverlaysToSources( - ctx context.Context, component components.Component, outputDir string, + component components.Component, outputDir string, macros map[string]string, ) ([]string, error) { - var macrosFileName string + macroOverlays, err := p.prepareMacroOverlays(component, outputDir, macros) + if err != nil { + return nil, err + } - macrosFilePath, err := p.writeMacrosFile(ctx, component, outputDir) + repackedArchives, err := p.applyOverlays(component, outputDir, macroOverlays) if err != nil { - return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", + return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } - if macrosFilePath != "" { - macrosFileName = filepath.Base(macrosFilePath) + return repackedArchives, nil +} + +func (p *sourcePreparerImpl) prepareMacroOverlays( + component components.Component, outputDir string, macros map[string]string, +) ([]projectconfig.ComponentOverlay, error) { + if p.withoutLockfile { + return synthesizeInlineMacroOverlays(macros), nil } - repackedArchives, err := p.applyOverlays(component, outputDir, macrosFileName) + macrosFilePath, err := p.writeMacrosFile(component, outputDir, macros) if err != nil { - return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", - component.GetName(), err) + return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", component.GetName(), err) } - return repackedArchives, nil + if macrosFilePath == "" { + return nil, nil + } + + overlays, err := synthesizeMacroLoadOverlays(filepath.Base(macrosFilePath)) + if err != nil { + return nil, fmt.Errorf("failed to compute macros load overlays:\n%w", err) + } + + return overlays, nil } // applyOverlays applies all overlays (user-defined and system-generated) to the // component sources. It returns the names of any archives that were repacked by // archive overlays. func (p *sourcePreparerImpl) applyOverlays( - component components.Component, sourcesDirPath, macrosFileName string, + component components.Component, sourcesDirPath string, macroOverlays []projectconfig.ComponentOverlay, ) ([]string, error) { event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) defer event.End() @@ -411,10 +496,7 @@ func (p *sourcePreparerImpl) applyOverlays( return nil, err } - allOverlays, err := p.collectOverlays(component, macrosFileName) - if err != nil { - return nil, fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err) - } + allOverlays := p.collectOverlays(component, macroOverlays) if len(allOverlays) == 0 { return nil, nil @@ -473,28 +555,20 @@ func (p *sourcePreparerImpl) applyArchiveOverlayGroup( } // collectOverlays gathers all overlays for a component into a single ordered slice: -// macros-load first, then user overlays, followed by check-skip and file-header overlays. +// macro definitions first, then user overlays, followed by check-skip and file-header overlays. func (p *sourcePreparerImpl) collectOverlays( - component components.Component, macrosFileName string, -) ([]projectconfig.ComponentOverlay, error) { + component components.Component, macroOverlays []projectconfig.ComponentOverlay, +) []projectconfig.ComponentOverlay { config := component.GetConfig() var allOverlays []projectconfig.ComponentOverlay - if macrosFileName != "" { - macroOverlays, err := synthesizeMacroLoadOverlays(macrosFileName) - if err != nil { - return nil, fmt.Errorf("failed to compute macros load overlays:\n%w", err) - } - - allOverlays = append(allOverlays, macroOverlays...) - } - + allOverlays = append(allOverlays, macroOverlays...) allOverlays = append(allOverlays, config.Overlays...) allOverlays = append(allOverlays, synthesizeCheckSkipOverlays(config.Build.Check)...) allOverlays = append(allOverlays, generateFileHeaderOverlay()...) - return allOverlays, nil + return allOverlays } // initSourcesRepo initializes a new git repository in sourcesDirPath, stages all files, @@ -543,6 +617,10 @@ func (p *sourcePreparerImpl) trySyntheticHistory( fingerprintConfig *projectconfig.ComponentConfig, sourcesDirPath string, ) error { + if p.skipSyntheticHistory { + return nil + } + config := component.GetConfig() componentName := component.GetName() @@ -775,7 +853,8 @@ func (p *sourcePreparerImpl) DiffSources( // Apply overlays in-place to the copied directory only. The repacked-archive // list is unused here: DiffSources diffs the trees directly and does not // rewrite a 'sources' file. - if _, err := p.applyOverlaysToSources(ctx, component, overlaidDir); err != nil { + macros := p.resolveMacros(ctx, component, overlaidDir, true) + if _, err := p.applyOverlaysToSources(component, overlaidDir, macros); err != nil { return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } @@ -1251,19 +1330,24 @@ func (p *sourcePreparerImpl) resolveSourceHash( return hash, hashType, nil } -// writeMacrosFile writes a macros file containing the resolved macros for a component. -// This includes with/without flags converted to macro format, and any explicit defines. -// If the build configuration produces no macros, no file is written and an empty path is -// returned. Otherwise, the path to the written macros file is returned. -func (p *sourcePreparerImpl) writeMacrosFile( - ctx context.Context, component components.Component, outputDir string, -) (string, error) { - macros := buildMacrosMap(component.GetConfig().Build) +func (p *sourcePreparerImpl) resolveMacros( + ctx context.Context, component components.Component, outputDir string, applyOverlays bool, +) map[string]string { + if !applyOverlays { + return nil + } - // Layer Fedora upstream provenance macros on top, reading the pristine - // upstream spec that is already on disk (before overlays are applied). + macros := buildMacrosMap(component.GetConfig().Build) p.addUpstreamProvenanceMacros(ctx, macros, component, outputDir) + return macros +} + +// writeMacrosFile writes the previously resolved macros, returning the file's path +// or an empty string when no macros are needed. +func (p *sourcePreparerImpl) writeMacrosFile( + component components.Component, outputDir string, macros map[string]string, +) (string, error) { contents := renderMacrosFile(macros) if contents == "" { return "", nil @@ -1363,6 +1447,34 @@ func renderMacrosFile(macros map[string]string) string { return strings.Join(lines, "\n") + "\n" } +func synthesizeInlineMacroOverlays(macros map[string]string) []projectconfig.ComponentOverlay { + if len(macros) == 0 { + return nil + } + + names := lo.Keys(macros) + slices.Sort(names) + + lines := []string{"# Build macros generated by azldev."} + + for _, name := range names { + value := macros[name] + if value == "" { + value = "%{nil}" + } + + // Lazy definitions retain macros-file semantics, including references to later spec tags. + lines = append(lines, fmt.Sprintf("%%define %s %s", name, value)) + } + + lines = append(lines, "") + + return []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlayPrependSpecLines, + Lines: lines, + }} +} + func synthesizeMacroLoadOverlays(macrosFileName string) ([]projectconfig.ComponentOverlay, error) { // Basic check that the macros file name is valid and doesn't require escaping. if strings.ContainsFunc(macrosFileName, func(r rune) bool { @@ -1465,7 +1577,7 @@ func (p *sourcePreparerImpl) applyOverlayList( } if err := ApplyOverlayToSources( - p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath, spec.WithEditor(p.specEditor), + p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath, p.specOptions()..., ); err != nil { return fmt.Errorf("failed to apply %#q overlay:\n%w", overlay.Type, err) } @@ -1474,6 +1586,15 @@ func (p *sourcePreparerImpl) applyOverlayList( return nil } +func (p *sourcePreparerImpl) specOptions() []spec.OpenOption { + options := []spec.OpenOption{spec.WithEditor(p.specEditor)} + if p.withoutLockfile || p.preserveReleaseFormatting { + options = append(options, spec.WithPreservedReleaseFormatting()) + } + + return options +} + func findSpecInDir( fs opctx.FS, component components.Component, dirPath string, ) (string, error) { diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 8276689c3..07139b171 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -4,12 +4,14 @@ package sources_test import ( + "context" "errors" "os" "path/filepath" "strings" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" @@ -27,6 +29,42 @@ import ( const testOutputDir = "/output" +func TestPrepareSources_FetchedSourcesRunBeforeOverlays(t *testing.T) { + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + ctx := testctx.NewCtx() + + component.EXPECT().GetName().AnyTimes().Return("curl") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Overlays: []projectconfig.ComponentOverlay{{ + Type: projectconfig.ComponentOverlayAppendSpecLines, + Lines: []string{"# overlay runs last"}, + }}, + }) + manager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(ctx.FS(), filepath.Join(dir, "curl.spec"), + []byte("Name: curl\nRelease: 1\n"), fileperms.PublicFile) + }) + preparer, err := sources.NewPreparer(manager, ctx.FS(), ctx, ctx, + sources.WithSkipLookaside(), + sources.WithFetchedSources(func(_ context.Context, _ components.Component, dir string) error { + path := filepath.Join(dir, "curl.spec") + content, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.NotContains(t, string(content), "overlay runs last") + + return fileutils.WriteFile(ctx.FS(), path, append(content, []byte("# release prepared\n")...), + fileperms.PublicFile) + })) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, component, testOutputDir, true)) + content, err := fileutils.ReadFile(ctx.FS(), testOutputDir+"/curl.spec") + require.NoError(t, err) + assert.Contains(t, string(content), "# release prepared\n# overlay runs last") +} + func TestNewPreparer(t *testing.T) { ctrl := gomock.NewController(t) sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) @@ -152,7 +190,9 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { component.EXPECT().GetName().AnyTimes().Return(componentName) component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{Defines: map[string]string{"prepare_fixture": "ready"}}, Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Summary", Value: "Overlaid"}, { Type: projectconfig.ComponentOverlayRemoveFile, Archive: archiveName, @@ -170,14 +210,26 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { }, }) - // The archive and 'sources' file already exist on disk; the source manager - // only needs to provide the spec file (FetchFiles is a no-op download). - sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).Return(nil) + // The archive and 'sources' file stand in for acquired upstream payloads. + sourceManager.EXPECT().FetchFiles(gomock.Any(), component, outputDir).DoAndReturn( + func(_ context.Context, _ components.Component, _ string) error { + content, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.NotContains(t, string(content), "%{load:") + assert.NotContains(t, string(content), "Summary: Overlaid") + + hash, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeMD5, archivePath) + require.NoError(t, err) + assert.Equal(t, originalHash, hash, "archive overlays must follow acquisition") + + return nil + }) + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, outputDir, gomock.Any()).DoAndReturn( func(_ interface{}, _ interface{}, dir string, _ ...sourceproviders.FetchComponentOption) error { return fileutils.WriteFile( ctx.FS(), filepath.Join(dir, componentName+".spec"), - []byte("# test spec"), fileperms.PublicFile) + []byte("Name: test-component\nVersion: 1.0\nRelease: 1\nSummary: Original\n"), fileperms.PublicFile) }, ) @@ -207,6 +259,15 @@ func TestPrepareSources_ArchiveOverlayRehashesSourcesEntry(t *testing.T) { assert.Equal(t, fileutils.HashTypeSHA512, entry.HashType, "configured post-overlay hash type must be used") assert.Equal(t, repackedHash, entry.Hash, "'sources' entry must record the repacked archive's hash") assert.NotEqual(t, originalHash, entry.Hash, "'sources' entry hash must have been updated") + + extractedDir := t.TempDir() + require.NoError(t, archive.ExtractAuto(archivePath, extractedDir)) + assert.FileExists(t, filepath.Join(extractedDir, "pkg-1.0/keep.txt")) + assert.NoFileExists(t, filepath.Join(extractedDir, "pkg-1.0/remove-me.txt")) + + specContent, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + assert.Contains(t, string(specContent), "Summary: Overlaid") } func TestPrepareSources_ArchiveOverlayAssociationValidation(t *testing.T) { @@ -344,12 +405,14 @@ func TestPrepareSources_WithSkipLookaside_SkipsFetchFiles(t *testing.T) { } assert.True(t, resolved.SkipLookaside, "FetchComponent should receive SkipLookaside option") + assert.True(t, resolved.SingleBranch, "lockless preparation should opt into single-branch cloning") return fileutils.WriteFile(ctx.FS(), outputSpecPath, []byte("# test spec"), fileperms.PublicFile) }, ) - preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx, sources.WithSkipLookaside()) + preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx, + sources.WithSkipLookaside(), sources.WithoutLockfileHistory()) require.NoError(t, err) err = preparer.PrepareSources(ctx, component, testOutputDir, true /*applyOverlays?*/) @@ -433,8 +496,87 @@ func TestPrepareSources_WritesMacrosFile(t *testing.T) { assert.Contains(t, specStr, "Source9999") } -// Tests for GenerateMacrosFileContents - these test content generation in isolation. +func TestPrepareSources_InlinesLocklessMacros(t *testing.T) { + for _, testCase := range []struct { + name string + skipOverlays bool + removeAll bool + }{ + {name: "configured macros"}, + {name: "all macros undefined", removeAll: true}, + {name: "overlays skipped", skipOverlays: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + ctx := testctx.NewCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + + config := projectconfig.ComponentConfig{ + Build: projectconfig.ComponentBuildConfig{ + With: []string{"feature", "removed"}, Without: []string{"legacy"}, + Defines: map[string]string{ + "empty": "", "late": "%{version}", "words": "two words", + }, + Undefines: []string{"_with_removed"}, + }, + } + if testCase.removeAll { + config.Build.Undefines = append(config.Build.Undefines, + "_with_feature", "_without_legacy", "empty", "late", "words") + } + + component.EXPECT().GetName().AnyTimes().Return("my-package") + component.EXPECT().GetConfig().AnyTimes().Return(&config) + manager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil) + specPath := filepath.Join(testOutputDir, "my-package.spec") + manager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, _ string, _ ...sourceproviders.FetchComponentOption) error { + return fileutils.WriteFile(ctx.FS(), specPath, []byte("Name: my-package\nVersion: 1\n"), + fileperms.PublicFile) + }) + + preparer, err := sources.NewPreparer(manager, ctx.FS(), ctx, ctx, sources.WithoutLockfileHistory()) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, component, testOutputDir, !testCase.skipOverlays)) + content, err := fileutils.ReadFile(ctx.FS(), specPath) + require.NoError(t, err) + + specText := string(content) + assert.NotContains(t, specText, "%{load:") + assert.NotContains(t, specText, "Source9999") + exists, err := fileutils.Exists(ctx.FS(), filepath.Join(testOutputDir, "my-package"+sources.MacrosFileExtension)) + require.NoError(t, err) + assert.False(t, exists) + + if testCase.removeAll || testCase.skipOverlays { + assert.NotContains(t, specText, "%define ") + + return + } + + last := -1 + + for _, line := range []string{ + "%define _with_feature 1\n", + "%define _without_legacy 1\n", + "%define empty %{nil}\n", + "%define late %{version}\n", + "%define words two words\n", + "Name: my-package\n", + } { + index := strings.Index(specText, line) + assert.Greater(t, index, last, "expected ordered inline definition %q", line) + last = index + } + + assert.NotContains(t, specText, "_with_removed") + }) + } +} + +// Tests for GenerateMacrosFileContents - these test content generation in isolation. func TestGenerateMacrosFileContents_EmptyConfig(t *testing.T) { contents := sources.GenerateMacrosFileContents(projectconfig.ComponentBuildConfig{}) diff --git a/internal/app/azldev/core/sources/upstream_provenance.go b/internal/app/azldev/core/sources/upstream_provenance.go index aa8b55578..d775f8f9a 100644 --- a/internal/app/azldev/core/sources/upstream_provenance.go +++ b/internal/app/azldev/core/sources/upstream_provenance.go @@ -35,11 +35,10 @@ const distPlaceholder = "%{?dist}" // against the Azure Linux buildroot's dist tag at build time. const distPlaceholderPlain = "%{dist}" -// autoreleaseResolver resolves an rpmautospec %autorelease to a concrete Fedora -// release number by running rpmautospec in an isolated environment (a mock -// chroot in production). Abstracted so tests can substitute a fake without a -// real mock. Implemented by [*MockProcessor]. -type autoreleaseResolver interface { +// AutoreleaseResolver resolves an rpmautospec %autorelease to a concrete Fedora +// release number. Lockless rendering runs it on the host; other workflows use +// [MockProcessor]. +type AutoreleaseResolver interface { // CalculateRelease returns the raw `rpmautospec calculate-release` output // for the spec named specFilename inside specHostDir (which must hold the // pristine .git history). The caller parses and validates the result. @@ -108,6 +107,13 @@ func (p *sourcePreparerImpl) addUpstreamProvenanceMacros( return } + _, versionDefined := macros[fedoraUpstreamVersionMacro] + + _, releaseDefined := macros[fedoraUpstreamReleaseMacro] + if versionDefined && releaseDefined { + return + } + specPath, err := findSpecInDir(p.fs, component, sourcesDir) if err != nil { // The user opted in via emit-upstream-provenance, so surface the @@ -131,6 +137,10 @@ func (p *sourcePreparerImpl) addUpstreamProvenanceMacros( setMacroIfAbsent(macros, fedoraUpstreamVersionMacro, version) } + if releaseDefined { + return + } + resolvedRelease := p.resolveUpstreamRelease(ctx, component.GetName(), release, sourcesDir, specPath) if resolvedRelease != "" { setMacroIfAbsent(macros, fedoraUpstreamReleaseMacro, resolvedRelease) @@ -185,8 +195,8 @@ func replaceDistTag(release, distTag string) string { } // calculateAutorelease resolves an rpmautospec %autorelease to the pristine -// Fedora release number (without the dist tag) by running rpmautospec inside a -// mock chroot via the configured [autoreleaseResolver]. Best-effort: returns "" +// Fedora release number (without the dist tag) through the configured +// [AutoreleaseResolver]. Best-effort: returns "" // with a warning when the dist-git history is unavailable, no resolver is // configured, the command fails, or the output is not a usable release number. func (p *sourcePreparerImpl) calculateAutorelease( @@ -202,7 +212,7 @@ func (p *sourcePreparerImpl) calculateAutorelease( } if p.autoreleaseResolver == nil { - slog.Warn("Skipping %fedora_upstream_release; no mock processor available to resolve %autorelease", + slog.Warn("Skipping %fedora_upstream_release; no resolver available to resolve %autorelease", "component", componentName) return "" diff --git a/internal/app/azldev/core/sources/upstream_provenance_internal_test.go b/internal/app/azldev/core/sources/upstream_provenance_internal_test.go index c0c60eae2..6f82405ac 100644 --- a/internal/app/azldev/core/sources/upstream_provenance_internal_test.go +++ b/internal/app/azldev/core/sources/upstream_provenance_internal_test.go @@ -9,7 +9,11 @@ import ( "path/filepath" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -26,6 +30,8 @@ type fakeAutoreleaseResolver struct { err error lastHostDir string lastSpecFile string + calls int + observe func(string, string) } func (f *fakeAutoreleaseResolver) CalculateRelease( @@ -34,6 +40,11 @@ func (f *fakeAutoreleaseResolver) CalculateRelease( f.lastHostDir = specHostDir f.lastSpecFile = specFilename + f.calls++ + if f.observe != nil { + f.observe(specHostDir, specFilename) + } + return f.output, f.err } @@ -354,6 +365,115 @@ func TestAddUpstreamProvenanceMacros_PlainDistTag(t *testing.T) { "unconditional %{dist} is expanded to the Fedora dist tag") } +func TestPrepareSources_CapturesPristineProvenance(t *testing.T) { + for _, testCase := range []struct { + name string + release string + override string + emit bool + applyOverlays bool + resolverErr error + wantRelease string + wantCalls int + }{ + { + name: "autorelease", release: "%autorelease -b 2", emit: true, applyOverlays: true, + wantRelease: "3.fc43", wantCalls: 1, + }, + { + name: "static release", release: "5%{?dist}", emit: true, applyOverlays: true, + wantRelease: "5.fc43", + }, + { + name: "explicit release override", release: "%autorelease", override: "42.fc43", + emit: true, applyOverlays: true, wantRelease: "42.fc43", + }, + { + name: "provenance disabled", release: "%autorelease", applyOverlays: true, + }, + { + name: "overlays disabled", release: "%autorelease", emit: true, + }, + { + name: "failed resolution is not retried", release: "%autorelease", emit: true, applyOverlays: true, + resolverErr: errors.New("release unavailable"), wantCalls: 1, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := testctx.NewCtx() + + config := upstreamCfg(testCase.emit) + if testCase.override != "" { + config.Build.Defines = map[string]string{fedoraUpstreamReleaseMacro: testCase.override} + } + + comp := mockComponent(ctrl, "fwupd-efi", config) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + manager.EXPECT().FetchComponent(gomock.Any(), comp, provenanceWorkDir, gomock.Any()).DoAndReturn( + func(_ context.Context, _ components.Component, dir string, _ ...sourceproviders.FetchComponentOption) error { + writeProvenanceSpec(t, ctx.FS(), "fwupd-efi", "1.6", testCase.release) + + return fileutils.WriteFile(ctx.FS(), filepath.Join(dir, ".git/HEAD"), + []byte("pinned upstream commit\n"), fileperms.PublicFile) + }) + + prepared := false + resolver := &fakeAutoreleaseResolver{ + output: "Calculated release number: 3\n", + err: testCase.resolverErr, + observe: func(dir, filename string) { + assert.False(t, prepared, "resolve provenance before release preparation") + + version, release, err := parseSpecVersionRelease(ctx.FS(), filepath.Join(dir, filename)) + require.NoError(t, err) + assert.Equal(t, "1.6", version) + assert.Equal(t, testCase.release, release) + + head, err := fileutils.ReadFile(ctx.FS(), filepath.Join(dir, ".git/HEAD")) + require.NoError(t, err) + assert.Equal(t, "pinned upstream commit\n", string(head)) + }, + } + preparer, err := NewPreparer(manager, ctx.FS(), ctx, ctx, + WithGitRepo(nil, nil, "43"), WithoutSyntheticHistory(), WithSkipLookaside(), + WithUpstreamProvenance(".fc43"), WithAutoreleaseResolver(resolver), + WithFetchedSources(func(_ context.Context, _ components.Component, _ string) error { + assert.Equal(t, testCase.wantCalls, resolver.calls) + + prepared = true + + writeProvenanceSpec(t, ctx.FS(), "fwupd-efi", "99", "99%{?dist}") + + return nil + })) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, comp, provenanceWorkDir, testCase.applyOverlays)) + assert.True(t, prepared) + assert.Equal(t, testCase.wantCalls, resolver.calls) + + path := filepath.Join(provenanceWorkDir, "fwupd-efi"+MacrosFileExtension) + if !testCase.emit || !testCase.applyOverlays { + exists, err := fileutils.Exists(ctx.FS(), path) + require.NoError(t, err) + assert.False(t, exists) + + return + } + + contents, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.Contains(t, string(contents), "%fedora_upstream_version 1.6\n") + + if testCase.wantRelease == "" { + assert.NotContains(t, string(contents), "%fedora_upstream_release ") + } else { + assert.Contains(t, string(contents), "%fedora_upstream_release "+testCase.wantRelease+"\n") + } + }) + } +} + // upstreamCfg returns a Fedora-upstream component config with provenance // emission toggled by emit. func upstreamCfg(emit bool) *projectconfig.ComponentConfig { diff --git a/internal/app/azldev/core/testutils/testenv.go b/internal/app/azldev/core/testutils/testenv.go index cf913c8ff..678da9fdf 100644 --- a/internal/app/azldev/core/testutils/testenv.go +++ b/internal/app/azldev/core/testutils/testenv.go @@ -71,7 +71,7 @@ func NewTestEnv(t *testing.T) *TestEnv { // NewTestEnvWithoutLockfile creates a [TestEnv] configured for azldev's // lock-file-free mode: no lock directory is configured and the environment // reports [azldev.Env.WithoutLockfile], so lock population and lock validation -// are skipped exactly as they are under the global '--without-lockfile' flag. +// are skipped for every component via an internal environment override. func NewTestEnvWithoutLockfile(t *testing.T) *TestEnv { t.Helper() diff --git a/internal/app/azldev/env.go b/internal/app/azldev/env.go index 9b426033f..35acd9f53 100644 --- a/internal/app/azldev/env.go +++ b/internal/app/azldev/env.go @@ -35,8 +35,8 @@ type EnvOptions struct { // WithoutLockfile selects the preview lock-file-free mode, in which // component state is tracked by generated upstream-commit config instead - // of per-component lock files. Set from the global '--without-lockfile' - // flag; false selects the default lock-file behavior. + // of per-component lock files. Used for internal historical comparisons; + // CLI operations select their modes from the project's component list. WithoutLockfile bool // Injected dependencies. @@ -283,6 +283,12 @@ func (env *Env) WithoutLockfile() bool { return env.withoutLockfile } +// ComponentWithoutLockfile reports the selected component's mode, honoring the +// internal whole-project override used by historical comparisons. +func (env *Env) ComponentWithoutLockfile(name string) bool { + return env.withoutLockfile || (env.config != nil && env.config.LocklessComponents[name]) +} + // SetEventListener registers the event listener to be used in this environment. func (env *Env) SetEventListener(eventListener opctx.EventListener) { env.eventListener = eventListener diff --git a/internal/app/azldev/env_test.go b/internal/app/azldev/env_test.go index 7aa094960..2485fc638 100644 --- a/internal/app/azldev/env_test.go +++ b/internal/app/azldev/env_test.go @@ -10,6 +10,7 @@ import ( "log/slog" "os" "runtime" + "strconv" "sync" "testing" "time" @@ -178,6 +179,23 @@ func TestEnvWithCancel(t *testing.T) { assert.NoError(t, original.Err()) } +func TestEnvComponentWithoutLockfile(t *testing.T) { + for _, global := range []bool{false, true} { + t.Run(strconv.FormatBool(global), func(t *testing.T) { + env := testutils.NewTestEnv(t) + if global { + env = testutils.NewTestEnvWithoutLockfile(t) + } + + env.Config.LocklessComponents = map[string]bool{"rust-asyncband": true} + + assert.True(t, env.Env.ComponentWithoutLockfile("rust-asyncband")) + assert.Equal(t, global, env.Env.ComponentWithoutLockfile("curl")) + assert.Equal(t, global, env.Env.WithoutLockfile()) + }) + } +} + func TestFixSuggestions(t *testing.T) { t.Run("no suggestions does not panic", func(t *testing.T) { testEnv := testutils.NewTestEnv(t) diff --git a/internal/lockfile/lockfile.go b/internal/lockfile/lockfile.go index 34c55236a..988e9690f 100644 --- a/internal/lockfile/lockfile.go +++ b/internal/lockfile/lockfile.go @@ -214,12 +214,18 @@ func validateConsistency( lockDir string, components map[string]projectconfig.ComponentConfig, checkOrphans bool, + ignoredComponents ...string, ) (missingOrStale, orphans []string, err error) { + ignored := make(map[string]bool, len(ignoredComponents)) + for _, name := range ignoredComponents { + ignored[name] = true + } + // Check each component that may need a lock file. Only skip components // that are explicitly local — unspecified source type inherits upstream // from distro defaults, so those need validation too. for name, comp := range components { - if comp.Spec.SourceType == projectconfig.SpecSourceTypeLocal { + if ignored[name] || comp.Spec.SourceType == projectconfig.SpecSourceTypeLocal { continue } diff --git a/internal/lockfile/lockfile_test.go b/internal/lockfile/lockfile_test.go index 485b51340..f02a83873 100644 --- a/internal/lockfile/lockfile_test.go +++ b/internal/lockfile/lockfile_test.go @@ -456,6 +456,31 @@ func TestValidateUpstreamCommit(t *testing.T) { } func TestValidateConsistency(t *testing.T) { + t.Run("ignored components skip validation but preserve orphan detection", func(t *testing.T) { + memFS := afero.NewMemMapFs() + store := lockfile.NewStore(memFS, testLockDir) + components := map[string]projectconfig.ComponentConfig{ + "missing": {}, + "corrupt": {}, + "stale": {Spec: projectconfig.SpecSource{UpstreamCommit: "new-commit"}}, + "locked": {}, + } + + require.NoError(t, fileutils.WriteFile( + memFS, mustLockPath(t, "corrupt"), []byte("not valid TOML"), fileperms.PublicFile, + )) + + lock := lockfile.New() + lock.UpstreamCommit = testCommitHash + require.NoError(t, store.Save("stale", lock)) + require.NoError(t, store.Save("orphan", lock)) + + stale, orphans, err := store.ValidateConsistency(components, true, "missing", "corrupt", "stale") + require.Error(t, err) + assert.Equal(t, []string{"locked"}, stale) + assert.Equal(t, []string{"orphan"}, orphans) + }) + t.Run("all valid", func(t *testing.T) { memFS := afero.NewMemMapFs() store := lockfile.NewStore(memFS, testLockDir) diff --git a/internal/lockfile/show.go b/internal/lockfile/show.go index afeca998f..002584632 100644 --- a/internal/lockfile/show.go +++ b/internal/lockfile/show.go @@ -65,6 +65,7 @@ func ReadAllAtCommit( repo *gogit.Repository, commitHash string, lockRelDir string, + ignoredComponents ...string, ) (map[string]ComponentLock, error) { lockRelDir, err := cleanLockRelDir(lockRelDir) if err != nil { @@ -80,7 +81,7 @@ func ReadAllAtCommit( return make(map[string]ComponentLock), nil } - return parseLockEntries(lockSubtree, lockRelDir, commitHash) + return parseLockEntries(lockSubtree, lockRelDir, commitHash, ignoredComponents...) } // cleanLockRelDir normalizes and validates a repo-relative lock directory path. @@ -136,9 +137,15 @@ func resolveLockSubtree( // parseLockEntries reads all lock files from a resolved tree. func parseLockEntries( lockSubtree *object.Tree, lockRelDir, commitHash string, + ignoredComponents ...string, ) (map[string]ComponentLock, error) { locks := make(map[string]ComponentLock) + ignored := make(map[string]bool, len(ignoredComponents)) + for _, name := range ignoredComponents { + ignored[name] = true + } + for _, entry := range lockSubtree.Entries { if !entry.Mode.IsFile() || strings.HasPrefix(entry.Name, ".") || @@ -147,6 +154,10 @@ func parseLockEntries( } name := strings.TrimSuffix(entry.Name, lockFileExtension) + if ignored[name] { + continue + } + entryPath := gopath.Join(lockRelDir, entry.Name) file, fileErr := lockSubtree.File(entry.Name) diff --git a/internal/lockfile/show_test.go b/internal/lockfile/show_test.go index 3ede47dd7..1ca7997b7 100644 --- a/internal/lockfile/show_test.go +++ b/internal/lockfile/show_test.go @@ -152,6 +152,19 @@ func TestShowAtCommit(t *testing.T) { func TestReadAllAtCommit(t *testing.T) { const lockDir = "locks" + t.Run("ignored corrupt locks are not read", func(t *testing.T) { + testFS := memfs.New() + repo, err := gogit.Init(memory.NewStorage(), testFS) + require.NoError(t, err) + commitFile(t, repo, testFS, "locks/curl.lock", validLockTOML, "add curl lock") + hash := commitFile(t, repo, testFS, "locks/rust-asyncband.lock", "invalid TOML", "add unused lock") + + locks, err := lockfile.ReadAllAtCommit(repo, hash, lockDir, "rust-asyncband") + require.NoError(t, err) + require.Len(t, locks, 1) + assert.Contains(t, locks, "curl") + }) + t.Run("reads all locks", func(t *testing.T) { testFS := memfs.New() repo, err := gogit.Init(memory.NewStorage(), testFS) diff --git a/internal/lockfile/store.go b/internal/lockfile/store.go index a031863ba..efa25fdb0 100644 --- a/internal/lockfile/store.go +++ b/internal/lockfile/store.go @@ -26,10 +26,12 @@ type LockReader interface { LockDir() string // ValidateConsistency checks lock files against the resolved component // configs. Returns sorted lists of components with missing/stale locks - // and orphan component names. + // and orphan component names. ignoredComponents are exempt from lock + // validation but still count as configured components for orphan detection. ValidateConsistency( components map[string]projectconfig.ComponentConfig, checkOrphans bool, + ignoredComponents ...string, ) (missingOrStale, orphans []string, err error) } @@ -210,8 +212,9 @@ func (s *Store) Remove(componentName string) error { func (s *Store) ValidateConsistency( components map[string]projectconfig.ComponentConfig, checkOrphans bool, + ignoredComponents ...string, ) (missingOrStale, orphans []string, err error) { - return validateConsistency(s.fs, s.lockDir, components, checkOrphans) + return validateConsistency(s.fs, s.lockDir, components, checkOrphans, ignoredComponents...) } // FindOrphanLockFiles returns component names that have lock files but no diff --git a/internal/projectconfig/config.go b/internal/projectconfig/config.go index f3d320bef..edabfa16b 100644 --- a/internal/projectconfig/config.go +++ b/internal/projectconfig/config.go @@ -18,9 +18,8 @@ import ( // for cleaning it up -- but not until after it is done using the loaded configuration. The loaded // configuration may implicitly depend on the contents of the temporary directory. // -// When withoutLockfile is set, component definitions merge with override semantics, -// are validated only once the whole project is assembled, and the project's lock -// directory is left unset. +// Lockless component definitions merge with override semantics and are validated +// once the whole project is assembled. The global mode overrides the component list. func LoadProjectConfig( fs opctx.FS, osEnv opctx.OSEnv, @@ -37,6 +36,11 @@ func LoadProjectConfig( return "", nil, fmt.Errorf("failed to find project root and config file:\n%w", err) } + locklessComponents, err := ReadLocklessComponents(fs, projectDir) + if err != nil { + return "", nil, err + } + configFilePaths := []string{} // Unless we were explicitly requested not to do so, copy our embedded default config files to a temporary @@ -87,6 +91,7 @@ func LoadProjectConfig( options := loadOptions{ permissiveConfigParsing: permissiveConfigParsing, withoutLockfile: withoutLockfile, + locklessComponents: locklessComponents, } config, err = loadAndResolveProjectConfig(fs, options, configFilePaths...) @@ -100,12 +105,5 @@ func LoadProjectConfig( // Apply project-relative defaults for any unset path fields. config.Project.ApplyProjectDefaults(projectDir) - // Lock-file-free mode never reads or writes lock files, so the lock directory - // is left unset. A project may still declare 'lock-dir' for compatibility with - // azldev's default mode; the value is simply ignored here. - if withoutLockfile { - config.Project.LockDir = "" - } - return projectDir, config, nil } diff --git a/internal/projectconfig/config_test.go b/internal/projectconfig/config_test.go index 5912b2c47..f807b8513 100644 --- a/internal/projectconfig/config_test.go +++ b/internal/projectconfig/config_test.go @@ -41,6 +41,86 @@ func writeProjectConfig(t *testing.T, ctx *testctx.TestCtx, contents string) { []byte(contents), fileperms.PublicFile)) } +func TestLoadProjectConfig_PerComponentLockModes(t *testing.T) { + ctx := newTestCtxWithXDGConfigHome() + writeProjectConfig(t, ctx, ` +includes = ["pins.toml", "components.toml"] + +[components.curl.spec] +type = "upstream" +upstream-commit = "aaa1111" + +[components.rust-asyncband.spec] +type = "upstream" +upstream-commit = "bbb1111" +`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "pins.toml"), []byte(` +[components.curl.spec] +upstream-commit = "aaa2222" +[components.rust-asyncband.spec] +upstream-commit = "bbb2222" +`), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "components.toml"), []byte(` +[components.curl.release] +calculation = "static" +[components.rust-asyncband.release] +calculation = "static" +`), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "lockless-components.txt"), + []byte("rust-asyncband\n"), fileperms.PublicFile)) + + for _, global := range []bool{false, true} { + _, config, err := projectconfig.LoadProjectConfig( + ctx.FS(), ctx.OSEnv(), testProjectDir, true, "/tmp", nil, false, global, + ) + require.NoError(t, err) + assert.Equal(t, map[string]bool{"rust-asyncband": true}, config.LocklessComponents) + rustComponent := config.Components["rust-asyncband"] + assert.Equal(t, "bbb2222", rustComponent.Spec.UpstreamCommit) + assert.Equal(t, filepath.Join(testProjectDir, "pins.toml"), + rustComponent.UpstreamCommitConfigFile().SourcePath()) + + curlComponent := config.Components["curl"] + assert.Equal(t, "aaa2222", curlComponent.Spec.UpstreamCommit) + + expectedSource := "components.toml" + if global { + expectedSource = "pins.toml" + } + + assert.Equal(t, filepath.Join(testProjectDir, expectedSource), + curlComponent.UpstreamCommitConfigFile().SourcePath()) + } +} + +func TestLoadProjectConfig_LockModesValidation(t *testing.T) { + for _, listed := range []string{"rust-asyncband", "curl"} { + t.Run(listed, func(t *testing.T) { + ctx := newTestCtxWithXDGConfigHome() + writeProjectConfig(t, ctx, ` +includes = ["later.toml"] +[components.rust-asyncband.release] +calculation = "invalid" +`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "later.toml"), []byte(` +[components.rust-asyncband.release] +calculation = "static" +`), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(testProjectDir, "lockless-components.txt"), + []byte(listed+"\n"), fileperms.PublicFile)) + + _, _, err := projectconfig.LoadProjectConfig( + ctx.FS(), ctx.OSEnv(), testProjectDir, true, "/tmp", nil, false, false, + ) + if listed == "rust-asyncband" { + require.NoError(t, err) + } else { + require.Error(t, err, "unlisted component definitions must still be validated per file") + } + }) + } +} + // TestLoadProjectConfig_NoUserConfig verifies that loading succeeds (and produces a // project-derived value) when no user-level config file is present. func TestLoadProjectConfig_NoUserConfig(t *testing.T) { @@ -173,9 +253,8 @@ description = "`+testUserDesc+`" assert.Equal(t, testUserDesc, config.Project.Description) } -// TestLoadProjectConfig_LockDirByMode verifies that the project's lock directory is -// defaulted in azldev's default mode and left unset in lock-file-free mode, where an -// explicitly configured 'lock-dir' is accepted but ignored. +// TestLoadProjectConfig_LockDirByMode verifies that the project's lock directory +// remains available in either global mode for per-component rendering. func TestLoadProjectConfig_LockDirByMode(t *testing.T) { testCases := []struct { name string @@ -183,7 +262,7 @@ func TestLoadProjectConfig_LockDirByMode(t *testing.T) { expected string }{ {name: "default mode", withoutLockfile: false, expected: filepath.Join(testProjectDir, "legacy-locks")}, - {name: "lock-file-free mode", withoutLockfile: true, expected: ""}, + {name: "lock-file-free mode", withoutLockfile: true, expected: filepath.Join(testProjectDir, "legacy-locks")}, } for _, testCase := range testCases { diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index ee6f07c73..c19beae03 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -127,13 +127,17 @@ func (f ConfigFile) Validate() error { return nil } -// validateNonComponentFields validates every field except the component -// definitions. Lock-file-free mode merges component definitions across config -// files with override semantics, so a single file may legitimately be -// incomplete; components are validated once the whole project is assembled. -func (f ConfigFile) validateNonComponentFields() error { +// validateForLockModes defers validation of partial lockless definitions until +// the project is assembled, while validating locked definitions immediately. +func (f ConfigFile) validateForLockModes(options loadOptions) error { components := f.Components - f.Components = nil + f.Components = make(map[string]ComponentConfig) + + for name, component := range components { + if !options.componentWithoutLockfile(name) { + f.Components[name] = component + } + } if err := f.Validate(); err != nil { return err @@ -142,6 +146,10 @@ func (f ConfigFile) validateNonComponentFields() error { // Source-file entries are atomic slice elements rather than partial nested // definitions, so validate them before their script paths become absolute. for componentName, component := range components { + if !options.componentWithoutLockfile(componentName) { + continue + } + if err := validateSourceFiles(component.SourceFiles, componentName, false); err != nil { return err } diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 79af6ca08..b0131a2d2 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -38,7 +38,12 @@ type loadOptions struct { // withoutLockfile selects lock-file-free mode, where component definitions are // merged with override semantics and validated only once fully assembled. - withoutLockfile bool + withoutLockfile bool + locklessComponents map[string]bool +} + +func (options loadOptions) componentWithoutLockfile(name string) bool { + return options.withoutLockfile || options.locklessComponents[name] } // Loads and resolves the project configuration files located at the given path. Referenced include files @@ -48,6 +53,7 @@ func loadAndResolveProjectConfig( fs opctx.FS, options loadOptions, configFilePaths ...string, ) (*ProjectConfig, error) { defaultConfig := NewProjectConfig() + defaultConfig.LocklessComponents = options.locklessComponents resolvedCfg := &defaultConfig for _, configFilePath := range configFilePaths { @@ -244,7 +250,23 @@ func mergeComponents(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile, options return mergeComponentsWithOverride(resolvedCfg, loadedCfg) } - return mergeComponentsAdditively(resolvedCfg, loadedCfg) + locked, lockless := *loadedCfg, *loadedCfg + locked.Components = make(map[string]ComponentConfig) + + lockless.Components = make(map[string]ComponentConfig) + for name, component := range loadedCfg.Components { + if options.componentWithoutLockfile(name) { + lockless.Components[name] = component + } else { + locked.Components[name] = component + } + } + + if err := mergeComponentsAdditively(resolvedCfg, &locked); err != nil { + return err + } + + return mergeComponentsWithOverride(resolvedCfg, &lockless) } // mergeComponentsAdditively merges component definitions from a loaded config file @@ -548,16 +570,10 @@ func loadProjectConfigFile( cfg.sourcePath = absFilePath cfg.dir = filepath.Dir(absFilePath) - // Make sure that the read data is internally consistent. In lock-file-free - // mode, component definitions support override merging across files, so they - // are validated only after the complete project configuration is assembled. + // Lockless component definitions support override merging across files, so + // validate them only after the complete configuration is assembled. // Syntax and unknown fields were already checked by the TOML decoder. - if options.withoutLockfile { - err = cfg.validateNonComponentFields() - } else { - err = cfg.Validate() - } - + err = cfg.validateForLockModes(options) if err != nil { return nil, err } diff --git a/internal/projectconfig/lockmodes.go b/internal/projectconfig/lockmodes.go new file mode 100644 index 000000000..445eca702 --- /dev/null +++ b/internal/projectconfig/lockmodes.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// ReadLocklessComponents reads the exact component names listed in +// 'lockless-components.txt' beside the project's top-level 'azldev.toml'. +func ReadLocklessComponents(fs opctx.FS, projectDir string) (map[string]bool, error) { + names := make(map[string]bool) + if projectDir == "" { + return names, nil + } + + path := filepath.Join(projectDir, "lockless-components.txt") + + info, err := fs.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return names, nil + } + + if err != nil { + return nil, fmt.Errorf("reading %#q:\n%w", path, err) + } + + if info.IsDir() { + return nil, fmt.Errorf("expected a file at %#q, found a directory", path) + } + + content, err := fileutils.ReadFile(fs, path) + if err != nil { + return nil, fmt.Errorf("reading %#q:\n%w", path, err) + } + + for line := range strings.SplitSeq(string(content), "\n") { + if name := strings.TrimSpace(line); name != "" { + names[name] = true + } + } + + return names, nil +} diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index 605066b80..61328c095 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -61,6 +61,8 @@ type ProjectConfig struct { RootConfigFilePath string `toml:"-" json:"-"` // Map from component names to groups they belong to; not serialized. GroupsByComponent map[string][]string `toml:"-" json:"-"` + // LocklessComponents is the project-root component selection; not serialized. + LocklessComponents map[string]bool `toml:"-" json:"-"` } // Constructs a default (empty) project configuration. @@ -86,23 +88,16 @@ func (cfg *ProjectConfig) Validate() error { return cfg.validate(false) } -// validate checks the assembled project configuration. In lock-file-free mode the -// component definitions are validated here rather than per config file, because -// override merging lets a single file hold a partial definition. +// validate checks the assembled project configuration. Lockless component +// definitions are validated here because a single file may be partial. func (cfg *ProjectConfig) validate(withoutLockfile bool) error { err := projectConfigValidator.Struct(cfg) if err != nil { return fmt.Errorf("config error:\n%w", err) } - if withoutLockfile { - if err := validateComponentStructs(cfg.Components); err != nil { - return err - } - - if err := validateComponentConfigs(cfg.Components, true); err != nil { - return err - } + if err := cfg.validateLocklessComponents(withoutLockfile); err != nil { + return err } if err := validateComponentGroupMembership(cfg.ComponentGroups, cfg.Components); err != nil { @@ -147,6 +142,22 @@ func (cfg *ProjectConfig) validate(withoutLockfile bool) error { return nil } +func (cfg *ProjectConfig) validateLocklessComponents(withoutLockfile bool) error { + lockless := make(map[string]ComponentConfig) + + for name, component := range cfg.Components { + if withoutLockfile || cfg.LocklessComponents[name] { + lockless[name] = component + } + } + + if err := validateComponentStructs(lockless); err != nil { + return err + } + + return validateComponentConfigs(lockless, true) +} + // validateImages runs the image-scoped validations: capabilities, architectures, // and SKU groups. func validateImages(cfg *ProjectConfig) error { diff --git a/internal/projectconfig/render.go b/internal/projectconfig/render.go index c623f8afc..0f377e6cd 100644 --- a/internal/projectconfig/render.go +++ b/internal/projectconfig/render.go @@ -6,10 +6,11 @@ package projectconfig // ComponentRenderConfig encapsulates configuration for rendering a component. type ComponentRenderConfig struct { // SkipFileFilter, when true, disables the post-render file filter for this - // component. Normally, rendered output is filtered to only include files + // locked component. Normally, locked rendered output is filtered to only include files // referenced by Source/Patch tags in the spec (as reported by spectool). // Some specs use macros that spectool cannot expand, causing referenced // files to be incorrectly removed. Setting this to true preserves all - // files from the dist-git checkout. - SkipFileFilter bool `toml:"skip-file-filter,omitempty" json:"skipFileFilter,omitempty" jsonschema:"title=Skip file filter,description=Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags"` + // files from the dist-git checkout. Lockless rendering always preserves all + // prepared files except '.git', regardless of this setting. + SkipFileFilter bool `toml:"skip-file-filter,omitempty" json:"skipFileFilter,omitempty" jsonschema:"title=Skip file filter,description=Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git."` } diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index f6b9b5c76..fd7edd45a 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -129,12 +129,17 @@ func (g *FedoraSourcesProviderImpl) GetComponent( defer fileutils.RemoveAllAndUpdateErrorIfNil(g.fs, tempDir, &err) // Clone the repository to temp directory with retry for transient network failures. + cloneOptions := []git.GitOptions{git.WithGitBranch(g.distroGitBranch)} + if resolved.SingleBranch && effectiveCommit == "" { + cloneOptions = append(cloneOptions, git.WithSingleBranch()) + } + err = retry.Do(ctx, g.retryConfig, func() error { // Clean up temp directory contents from any prior failed clone attempt. _ = g.fs.RemoveAll(tempDir) _ = fileutils.MkdirAll(g.fs, tempDir) - return g.gitProvider.Clone(ctx, gitRepoURL, tempDir, git.WithGitBranch(g.distroGitBranch)) + return g.gitProvider.Clone(ctx, gitRepoURL, tempDir, cloneOptions...) }) if err != nil { return fmt.Errorf("failed to clone git repository %#q:\n%w", gitRepoURL, err) @@ -308,6 +313,7 @@ func (g *FedoraSourcesProviderImpl) resolveCommit( return g.gitProvider.Clone(ctx, gitRepoURL, tempDir, git.WithGitBranch(g.distroGitBranch), + git.WithSingleBranch(), git.WithMetadataOnly(), git.WithQuiet(), ) diff --git a/internal/providers/sourceproviders/fedorasourceprovider_test.go b/internal/providers/sourceproviders/fedorasourceprovider_test.go index a9fa739de..839c7924d 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider_test.go +++ b/internal/providers/sourceproviders/fedorasourceprovider_test.go @@ -6,6 +6,9 @@ package sourceproviders_test import ( "context" "errors" + "os/exec" + "path/filepath" + "slices" "testing" "time" @@ -1104,3 +1107,68 @@ func TestGetComponent_LockedCommitTakesPriorityOverConfigPin(t *testing.T) { err = provider.GetComponent(context.Background(), mockComponent, destDir) require.NoError(t, err) } + +func TestGetComponent_SingleBranchPreservesPinnedCommits(t *testing.T) { + for _, testCase := range []struct { + name string + configPin string + lockedPin string + wantSingle bool + wantCommit string + }{ + {name: "floating", wantSingle: true, wantCommit: "branch-head"}, + {name: "explicit off-branch pin", configPin: testUpstreamCommit, wantCommit: testUpstreamCommit}, + {name: "locked off-branch pin", lockedPin: testUpstreamCommit, wantCommit: testUpstreamCommit}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + gitProvider, err := git.NewGitProviderImpl(env.EventListener, env.CmdFactory) + require.NoError(t, err) + ctrl := gomock.NewController(t) + extractor := fedorasource_test.NewMockFedoraSourceDownloader(ctrl) + provider, err := sourceproviders.NewFedoraSourcesProviderImpl( + env.FS(), env.DryRunnable, gitProvider, extractor, testResolvedDistro(), retry.Disabled()) + require.NoError(t, err) + + config := &projectconfig.ComponentConfig{ + Name: testPackageName, + Spec: projectconfig.SpecSource{UpstreamCommit: testCase.configPin}, + } + if testCase.lockedPin != "" { + config.Locked = &projectconfig.ComponentLockData{UpstreamCommit: testCase.lockedPin} + } + + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return(testPackageName) + component.EXPECT().GetConfig().AnyTimes().Return(config) + + cloned, checkedOut := false, false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if cmd.Args[1] == "clone" { + cloned = true + + assert.Equal(t, testCase.wantSingle, slices.Contains(cmd.Args, "--single-branch")) + cloneDir := cmd.Args[len(cmd.Args)-1] + + return fileutils.WriteFile(env.FS(), filepath.Join(cloneDir, testPackageName+".spec"), + []byte("Name: "+testPackageName), fileperms.PublicFile) + } + + checkedOut = true + + assert.Equal(t, "checkout", cmd.Args[3]) + assert.Equal(t, testCase.wantCommit, cmd.Args[4]) + + return nil + } + env.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + return "branch-head", nil + } + + require.NoError(t, provider.GetComponent(t.Context(), component, destDir, + sourceproviders.WithSingleBranch(), sourceproviders.WithSkipLookaside())) + assert.True(t, cloned) + assert.True(t, checkedOut) + }) + } +} diff --git a/internal/providers/sourceproviders/localcontentsprovider.go b/internal/providers/sourceproviders/localcontentsprovider.go index e17db5b5d..16b8931d9 100644 --- a/internal/providers/sourceproviders/localcontentsprovider.go +++ b/internal/providers/sourceproviders/localcontentsprovider.go @@ -15,6 +15,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" "github.com/spf13/afero" ) @@ -26,6 +27,7 @@ func FetchLocalComponent( dryRunnable opctx.DryRunnable, eventListener opctx.EventListener, fileSystem opctx.FS, component components.Component, destDirPath string, resolveRequiredFiles bool, + options ...FetchComponentOption, ) error { if dryRunnable == nil { return errors.New("dry runnable cannot be nil") @@ -57,7 +59,9 @@ func FetchLocalComponent( } // Copy all files from the source directory to the destination directory - err = copySourceDirectory(dryRunnable, fileSystem, sourceDirPath, destDirPath) + opts := resolveFetchComponentOptions(options) + + err = copySourceDirectory(dryRunnable, fileSystem, sourceDirPath, destDirPath, component, opts.SkipLocalSpecConfig) if err != nil { return fmt.Errorf("failed to copy source directory for component %#q:\n%w", component.GetName(), err) @@ -90,13 +94,27 @@ func acquireSourceDirPath(component components.Component) (string, error) { // copySourceDirectory copies all files from the source directory to the destination directory, // preserving file mode bits (e.g., executable bits for scripts). -func copySourceDirectory(dryRunnable opctx.DryRunnable, fs opctx.FS, sourceDirPath, destDirPath string) error { +func copySourceDirectory( + dryRunnable opctx.DryRunnable, fs opctx.FS, sourceDirPath, destDirPath string, + component components.Component, skipLocalSpecConfig bool, +) error { copyOptions := fileutils.CopyDirOptions{ CopyFileOptions: fileutils.CopyFileOptions{ PreserveFileMode: true, }, } + if skipLocalSpecConfig { + copyOptions.FileFilter = func(_ opctx.FS, destPath string) (bool, error) { + relative, err := filepath.Rel(destDirPath, destPath) + if err != nil { + return false, fmt.Errorf("resolving copied local file path:\n%w", err) + } + + return shouldCopyLocalSource(fs, component, filepath.Join(sourceDirPath, relative)) + } + } + err := fileutils.CopyDirRecursive(dryRunnable, fs, sourceDirPath, destDirPath, copyOptions) if err != nil { return fmt.Errorf("failed to copy files from %#q to %#q:\n%w", @@ -106,6 +124,42 @@ func copySourceDirectory(dryRunnable opctx.DryRunnable, fs opctx.FS, sourceDirPa return nil } +func shouldCopyLocalSource(fs opctx.FS, component components.Component, sourcePath string) (bool, error) { + if filepath.Ext(sourcePath) != ".toml" { + return true, nil + } + + content, err := fileutils.ReadFile(fs, sourcePath) + if err != nil { + return false, fmt.Errorf("reading local TOML file %#q:\n%w", sourcePath, err) + } + + var ( + config projectconfig.ConfigFile + localPath string + ) + + // Unrelated TOML sidecars need not parse as azldev configuration. + if toml.Unmarshal(content, &config) == nil { + localPath = config.Components[component.GetName()].Spec.Path + } + + if localPath == "" { + return true, nil + } + + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(filepath.Dir(sourcePath), localPath) + } + + specPath, err := component.GetSpec().GetPath() + if err != nil { + return false, fmt.Errorf("getting local spec path:\n%w", err) + } + + return filepath.Clean(localPath) != filepath.Clean(specPath), nil +} + // copyRequiredFiles resolves and copies any required files that may be stored separately // from the spec file's directory. func copyRequiredFiles( diff --git a/internal/providers/sourceproviders/localcontentsprovider_test.go b/internal/providers/sourceproviders/localcontentsprovider_test.go index 46c860806..132ee14dd 100644 --- a/internal/providers/sourceproviders/localcontentsprovider_test.go +++ b/internal/providers/sourceproviders/localcontentsprovider_test.go @@ -134,6 +134,44 @@ func TestFetchLocalComponent_NoRequiredFiles(t *testing.T) { require.Equal(t, testParams.specContents, string(contents)) } +func TestFetchLocalComponent_SkipDefiningSpecConfig(t *testing.T) { + for _, skip := range []bool{false, true} { + t.Run(map[bool]string{false: "ordinary fetch", true: "lockless render"}[skip], func(t *testing.T) { + params := setupLocalFetcherTest(t) + + files := map[string]string{ + "definition.toml": "[components.test.spec]\ntype = 'local'\npath = 'test.spec'\n", + "metadata.toml": "[components.test]\n", + "settings.toml": "setting = true\n", + "other.toml": "[components.other.spec]\ntype = 'local'\npath = 'test.spec'\n", + "overridden.toml": "[components.test.spec]\ntype = 'local'\npath = 'other.spec'\n", + "invalid.toml": "not valid TOML [\n", + "nested/def.toml": "[components.test.spec]\ntype = 'local'\npath = '../test.spec'\n", + } + for name, content := range files { + require.NoError(t, fileutils.WriteFile(params.env.TestFS, filepath.Join(params.specDir, name), + []byte(content), fileperms.PublicFile)) + } + + var options []sourceproviders.FetchComponentOption + if skip { + options = append(options, sourceproviders.WithSkipLocalSpecConfig()) + } + + require.NoError(t, sourceproviders.FetchLocalComponent(params.env.DryRunnable, + params.env.EventListener, params.env.TestFS, params.component, testOutputDir, false, options...)) + + for name := range files { + exists, err := fileutils.Exists(params.env.TestFS, filepath.Join(testOutputDir, name)) + require.NoError(t, err) + + want := !skip || (name != "definition.toml" && name != "nested/def.toml") + require.Equal(t, want, exists, name) + } + }) + } +} + func TestFetchLocalComponent_RequiredFileMissing(t *testing.T) { testParams := setupLocalFetcherTest(t) diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 4bf07d45a..53a8577ce 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -75,6 +75,9 @@ type FetchComponentOptions struct { // fetching. Git-tracked files (spec, patches, scripts) are still fetched from the // upstream clone. The sources manifest file remains available for validation. SkipLookaside bool + + SkipLocalSpecConfig bool + SingleBranch bool } // FetchComponentOption is a functional option for configuring component fetch behavior. @@ -96,6 +99,16 @@ func WithSkipLookaside() FetchComponentOption { } } +// WithSkipLocalSpecConfig excludes the TOML defining a local spec's path from fetched files. +func WithSkipLocalSpecConfig() FetchComponentOption { + return func(o *FetchComponentOptions) { o.SkipLocalSpecConfig = true } +} + +// WithSingleBranch restricts upstream cloning to the selected distro branch. +func WithSingleBranch() FetchComponentOption { + return func(o *FetchComponentOptions) { o.SingleBranch = true } +} + // resolveFetchComponentOptions applies all functional options and returns the resolved options. func resolveFetchComponentOptions(opts []FetchComponentOption) FetchComponentOptions { var resolved FetchComponentOptions @@ -705,7 +718,8 @@ func (m *sourceManager) resolveUpstreamSourceIdentity( func (m *sourceManager) fetchLocalComponent( ctx context.Context, component components.Component, destDirPath string, opts FetchComponentOptions, ) error { - err := FetchLocalComponent(m.dryRunnable, m.eventListener, m.fs, component, destDirPath, false) + err := FetchLocalComponent(m.dryRunnable, m.eventListener, m.fs, component, destDirPath, false, + func(o *FetchComponentOptions) { *o = opts }) if err != nil { return fmt.Errorf("failed to fetch local component %#q:\n%w", component.GetName(), err) diff --git a/internal/providers/sourceproviders/sourcemanager_test.go b/internal/providers/sourceproviders/sourcemanager_test.go index d7bf67d66..3e81fea09 100644 --- a/internal/providers/sourceproviders/sourcemanager_test.go +++ b/internal/providers/sourceproviders/sourcemanager_test.go @@ -522,12 +522,16 @@ func TestSourceManager_FetchComponent_LocalComponent_WithSkipLookaside(t *testin []byte("Name: test-component\nVersion: 1.0\n"), fileperms.PrivateFile)) require.NoError(t, fileutils.WriteFile(env.TestFS, specDir+"/sources", []byte("SHA512 (big-source.tar.gz) = abc123def456\n"), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(env.TestFS, specDir+"/definition.toml", + []byte("[components.test-component.spec]\ntype = 'local'\npath = 'test-component.spec'\n"), + fileperms.PublicFile)) sourceManager, err := sourceproviders.NewSourceManager(env.Env, testDefaultDistro()) require.NoError(t, err) // With SkipLookaside, downloadLookasideSources is not called and the fetch succeeds. - err = sourceManager.FetchComponent(t.Context(), component, testDestDir, sourceproviders.WithSkipLookaside()) + err = sourceManager.FetchComponent(t.Context(), component, testDestDir, + sourceproviders.WithSkipLookaside(), sourceproviders.WithSkipLocalSpecConfig()) require.NoError(t, err) // Spec was copied to destination (FetchLocalComponent still ran). @@ -535,6 +539,10 @@ func TestSourceManager_FetchComponent_LocalComponent_WithSkipLookaside(t *testin require.NoError(t, err) assert.True(t, specExists, "spec file should exist in destination") + configExists, err := fileutils.Exists(env.TestFS, filepath.Join(testDestDir, "definition.toml")) + require.NoError(t, err) + assert.False(t, configExists, "defining local spec TOML must not be copied") + // Source tarball was NOT downloaded (lookaside was skipped). tarballExists, err := fileutils.Exists(env.TestFS, filepath.Join(testDestDir, "big-source.tar.gz")) require.NoError(t, err) diff --git a/internal/rpm/mock/mock.go b/internal/rpm/mock/mock.go index 566aa4973..9a787d4d0 100644 --- a/internal/rpm/mock/mock.go +++ b/internal/rpm/mock/mock.go @@ -4,6 +4,7 @@ package mock import ( + "bytes" "context" "errors" "fmt" @@ -294,7 +295,7 @@ type CommonBuildOptions struct { // SRPMBuildOptions encapsulates options for building source RPMs using mock. type SRPMBuildOptions struct { CommonBuildOptions - // Add any SRPM-specific options here in the future if needed + RPMAutospec bool } // RPMBuildOptions encapsulates options for building binary RPMs using mock. @@ -324,6 +325,14 @@ func (r *Runner) BuildSRPM( "--resultdir", outputDirPath, ) + if options.RPMAutospec { + if err := r.requireRPMAutospec(ctx); err != nil { + return err + } + + args = append(args, "--enable-plugin", "rpmautospec") + } + // Add 'with'-style enabled conditionals. for _, with := range options.With { args = append(args, "--with", with) @@ -373,6 +382,31 @@ func (r *Runner) BuildSRPM( return nil } +func (r *Runner) requireRPMAutospec(ctx context.Context) error { + args := append(r.getBaseArgs(), "--enable-plugin", "rpmautospec", "--print-root-path") + raw := exec.CommandContext(ctx, MockBinary, args...) + + var stderr bytes.Buffer + + raw.Stderr = &stderr + + cmd, err := r.cmdFactory.Command(raw) + if err != nil { + return fmt.Errorf("checking mock rpmautospec plugin:\n%w", err) + } + + if _, err := cmd.RunAndGetOutput(ctx); err != nil { + return fmt.Errorf("checking mock rpmautospec plugin:\n%w\n%s", err, strings.TrimSpace(stderr.String())) + } + + if strings.Contains(stderr.String(), "rpmautospec plugin") && + strings.Contains(stderr.String(), "not installed") { + return errors.New("mock's rpmautospec plugin must be installed for lockless builds (requires mock 5.3 or newer)") + } + + return nil +} + // Given a path to a Source RPM (SRPM), invokes mock to build a binary RPM. func (r *Runner) BuildRPM(ctx context.Context, srpmPath, outputDirPath string, options RPMBuildOptions) error { slog.Debug("Building RPM", "SRPM", srpmPath) diff --git a/internal/rpm/mock/mock_test.go b/internal/rpm/mock/mock_test.go index 3d402c9ce..9601b84b7 100644 --- a/internal/rpm/mock/mock_test.go +++ b/internal/rpm/mock/mock_test.go @@ -6,6 +6,7 @@ package mock_test import ( "errors" "fmt" + "io" "os/exec" "path/filepath" "strings" @@ -145,6 +146,25 @@ func TestBuildSRPM(t *testing.T) { assert.Contains(t, mockCmd, "-r "+testMockConfigPath) } +func TestBuildSRPM_MissingRPMAutospecPlugin(t *testing.T) { + ctx := newTestCtxWithMockPrereqsPresent() + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + _, err := io.WriteString(cmd.Stderr, + "WARNING: rpmautospec plugin is enabled in configuration but is not installed\n") + + return testMockRootPath, err + } + ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + t.Fatal("must not build without the requested rpmautospec plugin") + + return nil + } + runner := mock.NewRunner(ctx, testMockConfigPath) + err := runner.BuildSRPM(ctx, testSpecPath, testSourceDirPath, testOutputDirPath, + mock.SRPMBuildOptions{RPMAutospec: true}) + require.ErrorContains(t, err, "rpmautospec plugin must be installed") +} + func TestBuildSRPM_MockFails(t *testing.T) { testError := errors.New("injected mock failure") @@ -158,6 +178,48 @@ func TestBuildSRPM_MockFails(t *testing.T) { require.ErrorIs(t, err, testError) } +func TestBuildSRPM_RPMAutospecProbeFailure(t *testing.T) { + ctx := newTestCtxWithMockPrereqsPresent() + testError := &exec.ExitError{} + + const diagnostic = "ERROR: could not load mock configuration" + + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + _, err := io.WriteString(cmd.Stderr, diagnostic+"\n") + require.NoError(t, err) + + return "", testError + } + ctx.CmdFactory.RunHandler = func(_ *exec.Cmd) error { + t.Fatal("must not build when the plugin probe fails") + + return nil + } + + runner := mock.NewRunner(ctx, testMockConfigPath) + err := runner.BuildSRPM(ctx, testSpecPath, testSourceDirPath, testOutputDirPath, + mock.SRPMBuildOptions{RPMAutospec: true}) + require.ErrorIs(t, err, testError) + require.ErrorContains(t, err, diagnostic) +} + +func TestBuildSRPM_RPMAutospec(t *testing.T) { + ctx := newTestCtxWithMockPrereqsPresent() + + var args []string + + ctx.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + args = cmd.Args + + return nil + } + runner := mock.NewRunner(ctx, testMockConfigPath) + require.NoError(t, runner.BuildSRPM(ctx, testSpecPath, testSourceDirPath, testOutputDirPath, + mock.SRPMBuildOptions{RPMAutospec: true})) + assert.Contains(t, strings.Join(args, " "), "--enable-plugin rpmautospec") + assert.Contains(t, strings.Join(args, " "), "--sources "+testSourceDirPath) +} + func TestBuildRPM(t *testing.T) { const ( withoutFeature = "without-feature" diff --git a/internal/rpm/spec/edit_test.go b/internal/rpm/spec/edit_test.go index 5fff1d154..32644691f 100644 --- a/internal/rpm/spec/edit_test.go +++ b/internal/rpm/spec/edit_test.go @@ -638,26 +638,96 @@ Name: value } func TestUpdateExistingTagUpdatesRepeatedConditionalTags(t *testing.T) { - specFile, err := spec.OpenSpec(strings.NewReader(`Name: example -%if 0 -Release: 1 -%else -Release: 2 -%endif -`), spec.WithEditor(spec.EditorStructural)) - require.NoError(t, err) + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + for _, formatting := range []struct { + name string + preserve bool + }{{name: "default"}, {name: "preserved", preserve: true}} { + t.Run(formatting.name, func(t *testing.T) { + for _, method := range []string{"update", "set"} { + t.Run(method, func(t *testing.T) { + input := []string{ + "Name: example", "%if 0", "release: 1", "%else", "Release:\t 2", "%endif", + } + + options := []spec.OpenOption{spec.WithEditor(editor)} + if formatting.preserve { + options = append(options, spec.WithPreservedReleaseFormatting()) + } + + specFile, err := spec.OpenSpec(strings.NewReader(strings.Join(input, "\n")+"\n"), options...) + require.NoError(t, err) + + if method == "update" { + err = specFile.UpdateExistingTag("", "Release", "3%{?dist}") + } else { + err = specFile.SetTag("", "Release", "3%{?dist}") + } + + require.NoError(t, err) + + var actual bytes.Buffer + require.NoError(t, specFile.Serialize(&actual)) + + input[2], input[4] = "Release: 3%{?dist}", "Release: 3%{?dist}" + if formatting.preserve { + input[2], input[4] = "release: 3%{?dist}", "Release:\t 3%{?dist}" + } + + assert.Equal(t, strings.Join(input, "\n")+"\n", actual.String()) + }) + } + }) + } + }) + } +} - require.NoError(t, specFile.UpdateExistingTag("", "Release", "3%{?dist}")) +func TestUpdateExistingTagFuncUpdatesEachTag(t *testing.T) { + errUpdate := errors.New("update failed") - var actual bytes.Buffer - require.NoError(t, specFile.Serialize(&actual)) - assert.Equal(t, `Name: example -%if 0 -Release: 3%{?dist} -%else -Release: 3%{?dist} -%endif -`, actual.String()) + for _, editor := range []spec.EditorMode{spec.EditorLegacy, spec.EditorStructural} { + t.Run(string(editor), func(t *testing.T) { + input := []string{ + "Name: example", "%if 0", "Release: 1", "%else", "Release:\t 2.%{snap}", "%endif", + } + + specFile, err := spec.OpenSpec(strings.NewReader(strings.Join(input, "\n")+"\n"), + spec.WithEditor(editor), spec.WithPreservedReleaseFormatting()) + require.NoError(t, err) + + releases, err := specFile.GetTags("", "Release") + require.NoError(t, err) + assert.Equal(t, []string{"1", "2.%{snap}"}, releases) + + var seen []string + + require.NoError(t, specFile.UpdateExistingTagFunc("", "Release", func(current string) (string, error) { + seen = append(seen, current) + + return "1" + current, nil + })) + assert.Equal(t, []string{"1", "2.%{snap}"}, seen) + + var actual bytes.Buffer + require.NoError(t, specFile.Serialize(&actual)) + + input[2], input[4] = "Release: 11", "Release:\t 12.%{snap}" + assert.Equal(t, strings.Join(input, "\n")+"\n", actual.String()) + + err = specFile.UpdateExistingTagFunc("", "Release", func(string) (string, error) { + return "", errUpdate + }) + require.ErrorIs(t, err, errUpdate) + + _, err = specFile.GetTags("", "Epoch") + require.ErrorIs(t, err, spec.ErrNoSuchTag) + require.ErrorIs(t, specFile.UpdateExistingTagFunc("", "Epoch", func(string) (string, error) { + return "1", nil + }), spec.ErrNoSuchTag) + }) + } } func TestRemoveTag(t *testing.T) { diff --git a/internal/rpm/spec/editor.go b/internal/rpm/spec/editor.go index 78967de13..73381ad08 100644 --- a/internal/rpm/spec/editor.go +++ b/internal/rpm/spec/editor.go @@ -6,6 +6,7 @@ package spec import ( "fmt" "io" + "strings" "time" ) @@ -22,7 +23,8 @@ const ( ) type editorOptions struct { - mode EditorMode + mode EditorMode + preserveReleaseFormatting bool } // OpenOption configures [OpenSpec]. @@ -35,6 +37,14 @@ func WithEditor(mode EditorMode) OpenOption { } } +// WithPreservedReleaseFormatting retains existing Release tag casing and spacing +// when updating its value. +func WithPreservedReleaseFormatting() OpenOption { + return func(options *editorOptions) { + options.preserveReleaseFormatting = true + } +} + //nolint:interfacebloat,inamedparam // The facade must cover the established public Spec API. type specEditor interface { Serialize(io.Writer) error @@ -47,6 +57,7 @@ type specEditor interface { VisitTagsPackage(string, func(*TagLine, *Context) error) error SetTag(string, string, string) error UpdateExistingTag(string, string, string) error + UpdateExistingTagFunc(string, string, func(string) (string, error)) error RemoveTag(string, string, string) error RemoveTagsMatching(string, func(string, string) bool) (int, error) AddTag(string, string, string) error @@ -65,6 +76,7 @@ type specEditor interface { RemoveSubpackage(string) error GetTag(string, string) (string, error) GetLastTag(string, string) (string, error) + GetTags(string, string) ([]string, error) } // Spec is the public facade for a configuration-selected RPM spec editor. @@ -72,6 +84,19 @@ type Spec struct { editor specEditor } +func updatedTagLine(original, tag, value string, preserveReleaseFormatting bool) string { + if preserveReleaseFormatting && strings.EqualFold(tag, "Release") { + prefix, rest, found := strings.Cut(original, ":") + if found { + spacing := rest[:len(rest)-len(strings.TrimLeft(rest, " \t"))] + + return prefix + ":" + spacing + value + } + } + + return fmt.Sprintf("%s: %s", tag, value) +} + func (s *Spec) activeEditor() specEditor { if s.editor == nil { s.editor = &legacySpec{} @@ -95,9 +120,9 @@ func OpenSpec(reader io.Reader, options ...OpenOption) (*Spec, error) { switch config.mode { case EditorLegacy, "": - editor, err = openLegacySpec(reader) + editor, err = openLegacySpec(reader, config.preserveReleaseFormatting) case EditorStructural: - editor, err = openStructuralSpec(reader) + editor, err = openStructuralSpec(reader, config.preserveReleaseFormatting) default: return nil, fmt.Errorf("unknown spec editor %#q", config.mode) } @@ -167,6 +192,15 @@ func (s *Spec) UpdateExistingTag(pkg, tag, value string) error { return s.activeEditor().UpdateExistingTag(pkg, tag, value) } +// UpdateExistingTagFunc updates every matching tag in the specified package to the +// value returned by update for that tag's current value. It stops at the first +// error returned by update. Returns [ErrNoSuchTag] if no matching tag exists. +// +//nolint:wrapcheck // Preserve errors returned by the selected editor. +func (s *Spec) UpdateExistingTagFunc(pkg, tag string, update func(current string) (string, error)) error { + return s.activeEditor().UpdateExistingTagFunc(pkg, tag, update) +} + // RemoveTag removes matching tags from the specified package. // //nolint:wrapcheck // Preserve errors returned by the selected editor. @@ -278,6 +312,14 @@ func (s *Spec) GetTag(pkg, tag string) (string, error) { return s.activeEditor().GetTag(pkg, tag) } +// GetTags returns the values of every matching tag in the specified package, in +// spec order. Returns [ErrNoSuchTag] if no matching tag exists. +// +//nolint:wrapcheck // Preserve errors returned by the selected editor. +func (s *Spec) GetTags(pkg, tag string) ([]string, error) { + return s.activeEditor().GetTags(pkg, tag) +} + // GetLastTag returns the final matching tag in the specified package. // //nolint:wrapcheck // Preserve errors returned by the selected editor. diff --git a/internal/rpm/spec/legacy_edit.go b/internal/rpm/spec/legacy_edit.go index 3f530beb6..be45931a4 100644 --- a/internal/rpm/spec/legacy_edit.go +++ b/internal/rpm/spec/legacy_edit.go @@ -48,25 +48,48 @@ func (s *legacySpec) SetTag(packageName string, tag string, value string) (err e // UpdateExistingTag looks for the first instance of the named tag in the given package; if it // finds such a tag instance, it replaces its value with the provided value. If no such tag // exists, it returns an error. -func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value string) (err error) { +func (s *legacySpec) UpdateExistingTag(packageName string, tag string, value string) error { slog.Debug("Updating tag in spec", "package", packageName, "tag", tag, "newValue", value) + return s.UpdateExistingTagFunc(packageName, tag, func(string) (string, error) { return value, nil }) +} + +// UpdateExistingTagFunc replaces the value of every instance of the named tag in the given +// package with the value returned by update for that instance's current value. If no such +// tag exists, it returns an error. +func (s *legacySpec) UpdateExistingTagFunc( + packageName string, tag string, update func(current string) (string, error), +) (err error) { tagToCompareAgainst := strings.ToLower(tag) - var updated bool + var ( + updated bool + updateErr error + ) err = s.VisitTagsPackage(packageName, func(tagLine *TagLine, ctx *Context) error { if strings.ToLower(tagLine.Tag) != tagToCompareAgainst { return nil } - ctx.ReplaceLine(fmt.Sprintf("%s: %s", tag, value)) + var value string + + value, updateErr = update(tagLine.Value) + if updateErr != nil { + return updateErr + } + + ctx.ReplaceLine(updatedTagLine(ctx.Target.Line.Text, tag, value, s.preserveReleaseFormatting)) updated = true return nil }) + if updateErr != nil { + return updateErr + } + if !updated { return fmt.Errorf("tag %#q not found in spec:\n%w", tag, ErrNoSuchTag) } @@ -1166,6 +1189,28 @@ func (s *legacySpec) GetTag(packageName, tag string) (string, error) { return value, nil } +// GetTags returns the values of all matching tags in a package, in spec order. +func (s *legacySpec) GetTags(packageName, tag string) ([]string, error) { + var values []string + + err := s.VisitTagsPackage(packageName, func(tagLine *TagLine, _ *Context) error { + if strings.EqualFold(tagLine.Tag, tag) { + values = append(values, tagLine.Value) + } + + return nil + }) + if err != nil { + return nil, err + } + + if len(values) == 0 { + return nil, fmt.Errorf("tag %#q not found in package %#q:\n%w", tag, packageName, ErrNoSuchTag) + } + + return values, nil +} + // GetLastTag returns the last matching tag in a package. func (s *legacySpec) GetLastTag(packageName, tag string) (string, error) { var value string diff --git a/internal/rpm/spec/legacy_spec.go b/internal/rpm/spec/legacy_spec.go index 7c3aaabc0..bcaae26a2 100644 --- a/internal/rpm/spec/legacy_spec.go +++ b/internal/rpm/spec/legacy_spec.go @@ -13,7 +13,8 @@ import ( ) type legacySpec struct { - rawLines []string + rawLines []string + preserveReleaseFormatting bool } type parseState struct { @@ -32,9 +33,9 @@ func newParseState() parseState { // OpenSpec reads in the contents of an RPM spec file from the provided reader, returning a [Spec] object. // An error is returned if the reader cannot be fully read (e.g., I/O error or line exceeds buffer size). -func openLegacySpec(reader io.Reader) (*legacySpec, error) { +func openLegacySpec(reader io.Reader, preserveReleaseFormatting bool) (*legacySpec, error) { scanner := bufio.NewScanner(reader) - spec := &legacySpec{} + spec := &legacySpec{preserveReleaseFormatting: preserveReleaseFormatting} // Read each line from the reader, parsing as we go. Store all parsed lines in the spec object. for scanner.Scan() { diff --git a/internal/rpm/spec/structural_edit.go b/internal/rpm/spec/structural_edit.go index 2b8dc2805..fb71dffea 100644 --- a/internal/rpm/spec/structural_edit.go +++ b/internal/rpm/spec/structural_edit.go @@ -34,9 +34,18 @@ func (s *structuralSpec) SetTag(packageName string, tag string, value string) (e // UpdateExistingTag replaces every instance of the named tag in the given // package with the provided value. If no such tag exists, it returns an error. -func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value string) (err error) { +func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value string) error { slog.Debug("Updating tag in spec", "package", packageName, "tag", tag, "newValue", value) + return s.UpdateExistingTagFunc(packageName, tag, func(string) (string, error) { return value, nil }) +} + +// UpdateExistingTagFunc replaces the value of every instance of the named tag in the given +// package with the value returned by update for that instance's current value. If no such +// tag exists, it returns an error. +func (s *structuralSpec) UpdateExistingTagFunc( + packageName string, tag string, update func(current string) (string, error), +) (err error) { tagToCompareAgainst := strings.ToLower(tag) var updated bool @@ -47,12 +56,17 @@ func (s *structuralSpec) UpdateExistingTag(packageName string, tag string, value return nil } - parsedTag, _, isTag := parseTagLine(line.Text) + parsedTag, current, isTag := parseTagLine(line.Text) if !isTag || strings.ToLower(parsedTag) != tagToCompareAgainst { return nil } - line.Replace(fmt.Sprintf("%s: %s", tag, value)) + value, updateErr := update(current) + if updateErr != nil { + return updateErr + } + + line.Replace(updatedTagLine(line.Text, tag, value, s.preserveReleaseFormatting)) updated = true @@ -192,6 +206,27 @@ func (s *structuralSpec) GetTag(packageName string, tag string) (string, error) return foundValue, nil } +// GetTags returns the values of all instances of the named tag in the given +// package, in spec order. Returns [ErrNoSuchTag] if the tag does not exist. +func (s *structuralSpec) GetTags(packageName string, tag string) ([]string, error) { + var values []string + + err := s.inspectTree(func(tree *specTree) error { + values = tree.GetTags(packageName, tag) + + return nil + }) + if err != nil { + return nil, err + } + + if len(values) == 0 { + return nil, fmt.Errorf("tag %#q not found in package %#q:\n%w", tag, packageName, ErrNoSuchTag) + } + + return values, nil +} + // GetLastTag returns the value of the final lexical instance of the named tag // in the given package. Returns [ErrNoSuchTag] if the tag does not exist. func (s *structuralSpec) GetLastTag(packageName string, tag string) (string, error) { diff --git a/internal/rpm/spec/structural_spec.go b/internal/rpm/spec/structural_spec.go index 73cff841d..8731e9ac6 100644 --- a/internal/rpm/spec/structural_spec.go +++ b/internal/rpm/spec/structural_spec.go @@ -52,7 +52,8 @@ var sectionTypesByName = map[string]SectionType{ // Spec encapsulates the contents of an RPM spec file. type structuralSpec struct { - rawLines []string + rawLines []string + preserveReleaseFormatting bool } // Line represents a single line in an RPM spec file. @@ -137,9 +138,9 @@ func (*RawLine) GetType() ParsedLineType { // OpenSpec reads in the contents of an RPM spec file from the provided reader, returning a [Spec] object. // An error is returned if the reader cannot be fully read (e.g., I/O error or line exceeds buffer size). -func openStructuralSpec(reader io.Reader) (*structuralSpec, error) { +func openStructuralSpec(reader io.Reader, preserveReleaseFormatting bool) (*structuralSpec, error) { scanner := bufio.NewScanner(reader) - spec := &structuralSpec{} + spec := &structuralSpec{preserveReleaseFormatting: preserveReleaseFormatting} // Read each line from the reader, parsing as we go. Store all parsed lines in the spec object. for scanner.Scan() { diff --git a/internal/rpm/spec/structural_tree_api.go b/internal/rpm/spec/structural_tree_api.go index 5d775421b..82ca03b1b 100644 --- a/internal/rpm/spec/structural_tree_api.go +++ b/internal/rpm/spec/structural_tree_api.go @@ -97,10 +97,17 @@ func (t *specTree) GetTag(pkg, name string) (string, bool) { // GetLastTag returns the last lexical tag matching name in the requested package. func (t *specTree) GetLastTag(pkg, name string) (string, bool) { - var ( - value string - found bool - ) + values := t.GetTags(pkg, name) + if len(values) == 0 { + return "", false + } + + return values[len(values)-1], true +} + +// GetTags returns the values of all instances of the named tag in pkg, in lexical order. +func (t *specTree) GetTags(pkg, name string) []string { + var values []string _ = t.VisitAllLines(func(secName, secPkg string, line *lineHandle) error { if secPkg != pkg || !isTagBearingSection(secName) { @@ -109,14 +116,13 @@ func (t *specTree) GetLastTag(pkg, name string) (string, bool) { tag, tagValue, isTag := parseTagLine(line.Text) if isTag && strings.EqualFold(tag, name) { - value = tagValue - found = true + values = append(values, tagValue) } return nil }) - return value, found + return values } // HasSection reports whether a section with name is present for any package. diff --git a/internal/upstreamcommit/store.go b/internal/upstreamcommit/store.go index 434900290..907108f4b 100644 --- a/internal/upstreamcommit/store.go +++ b/internal/upstreamcommit/store.go @@ -6,6 +6,7 @@ package upstreamcommit import ( + "bytes" "errors" "fmt" "log/slog" @@ -162,6 +163,10 @@ func (s *Store) Save(componentName, upstreamCommit string) error { return fmt.Errorf("serializing upstream commit TOML %#q:\n%w", path, err) } + // Only the spec table has values; omit the encoder's two empty parent headers. + _, data, _ = bytes.Cut(data, []byte("\n")) + _, data, _ = bytes.Cut(data, []byte("\n")) + data = append([]byte(generatedFileHeader), data...) if err := fileutils.MkdirAll(s.fs, s.dir); err != nil { @@ -176,9 +181,11 @@ func (s *Store) Save(componentName, upstreamCommit string) error { } // FindOrphans returns generated component TOMLs that do not correspond to an -// upstream component in components. +// upstream component in components. Configured ignoredComponents are preserved +// regardless of their source type. func (s *Store) FindOrphans( components map[string]projectconfig.ComponentConfig, + ignoredComponents ...string, ) ([]string, error) { entries, err := fileutils.ReadDir(s.fs, s.dir) if err != nil { @@ -194,6 +201,11 @@ func (s *Store) FindOrphans( return nil, fmt.Errorf("reading upstream commit TOML directory %#q:\n%w", s.dir, err) } + ignored := make(map[string]bool, len(ignoredComponents)) + for _, name := range ignoredComponents { + ignored[name] = true + } + var orphans []string for _, entry := range entries { @@ -205,7 +217,7 @@ func (s *Store) FindOrphans( name := strings.TrimSuffix(entry.Name(), fileExtension) component, ok := components[name] - if !ok || component.Spec.SourceType != projectconfig.SpecSourceTypeUpstream { + if !ok || (!ignored[name] && component.Spec.SourceType != projectconfig.SpecSourceTypeUpstream) { orphans = append(orphans, name) } } @@ -216,8 +228,10 @@ func (s *Store) FindOrphans( } // PruneOrphans removes generated TOMLs that are no longer needed. -func (s *Store) PruneOrphans(components map[string]projectconfig.ComponentConfig) (int, error) { - orphans, err := s.FindOrphans(components) +func (s *Store) PruneOrphans( + components map[string]projectconfig.ComponentConfig, ignoredComponents ...string, +) (int, error) { + orphans, err := s.FindOrphans(components, ignoredComponents...) if err != nil { return 0, err } diff --git a/internal/upstreamcommit/store_test.go b/internal/upstreamcommit/store_test.go index 923d56bd1..7bebc5735 100644 --- a/internal/upstreamcommit/store_test.go +++ b/internal/upstreamcommit/store_test.go @@ -5,6 +5,7 @@ package upstreamcommit_test import ( "path/filepath" + "strings" "testing" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" @@ -29,8 +30,6 @@ func TestStoreSaveAndGet(t *testing.T) { assert.Equal(t, `# This file was generated by 'azldev component refresh-upstream-commit' # Do not edit this file, changes will be lost # For more details see 'azldev component refresh-upstream-commit --help' -[components] -[components.bash] [components.bash.spec] upstream-commit = 'abcdef1234567' `, string(data)) @@ -47,6 +46,27 @@ upstream-commit = 'abcdef1234567' assert.Equal(t, "1234567abcdef", commit) } +func TestStoreSaveSingleTableWithQuotedComponentNames(t *testing.T) { + for _, name := range []string{"rust-asyncband", "libstdc++", "python3.12", `component"quote`} { + t.Run(name, func(t *testing.T) { + ctx := testctx.NewCtx() + store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) + require.NoError(t, store.Save(name, "abcdef1234567")) + path, err := store.Path(name) + require.NoError(t, err) + data, err := fileutils.ReadFile(ctx.FS(), path) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(string(data), "\n[")) + assert.Contains(t, string(data), ".spec]\n") + + commit, exists, err := store.Get(name) + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "abcdef1234567", commit) + }) + } +} + func TestStoreSaveReplacesExistingGeneratedContent(t *testing.T) { ctx := testctx.NewCtx() store := upstreamcommit.NewStore(ctx.FS(), testStoreDir) @@ -54,6 +74,8 @@ func TestStoreSaveReplacesExistingGeneratedContent(t *testing.T) { require.NoError(t, fileutils.MkdirAll(ctx.FS(), testStoreDir)) require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(` +[components] +[components.bash] [components.bash.spec] type = "local" path = "bash.spec" @@ -63,6 +85,11 @@ upstream-commit = "oldcommit" defines = { stale = "value" } `), fileperms.PublicFile)) + commit, exists, err := store.Get("bash") + require.NoError(t, err) + assert.True(t, exists) + assert.Equal(t, "oldcommit", commit) + require.NoError(t, store.Save("bash", "abcdef1234567")) data, err := fileutils.ReadFile(ctx.FS(), path) @@ -71,6 +98,7 @@ defines = { stale = "value" } assert.NotContains(t, string(data), "path") assert.NotContains(t, string(data), "defines") assert.Contains(t, string(data), "upstream-commit = 'abcdef1234567'") + assert.Equal(t, 1, strings.Count(string(data), "\n[")) } func TestStoreGetMissing(t *testing.T) { diff --git a/internal/utils/dirdiff/dirdiff.go b/internal/utils/dirdiff/dirdiff.go index ed2bb5360..3dd728109 100644 --- a/internal/utils/dirdiff/dirdiff.go +++ b/internal/utils/dirdiff/dirdiff.go @@ -7,6 +7,7 @@ package dirdiff import ( "bytes" "fmt" + "maps" "os" "path/filepath" "slices" @@ -29,12 +30,16 @@ import ( func DiffDirs(fs opctx.FS, dirA, dirB string, opts ...DiffOption) (*DiffResult, error) { cfg := newDiffConfig(opts) - filesA, err := collectFiles(fs, dirA) + ignoredOriginal := make(map[string]bool, len(cfg.ignoredPaths)+len(cfg.ignoredOriginalPaths)) + maps.Copy(ignoredOriginal, cfg.ignoredPaths) + maps.Copy(ignoredOriginal, cfg.ignoredOriginalPaths) + + filesA, err := collectFiles(fs, dirA, ignoredOriginal) if err != nil { return nil, fmt.Errorf("failed to walk original directory %#q:\n%w", dirA, err) } - filesB, err := collectFiles(fs, dirB) + filesB, err := collectFiles(fs, dirB, cfg.ignoredPaths) if err != nil { return nil, fmt.Errorf("failed to walk modified directory %#q:\n%w", dirB, err) } @@ -122,7 +127,7 @@ func mergedSortedKeys(filesA, filesB map[string]fileEntry) []string { // [afero.MemMapFs] does not support symlinks and will report them as regular files. // [afero.Walk] uses LstatIfPossible internally, so on supported filesystems symlinks // are correctly detected via [os.ModeSymlink] rather than being followed. -func collectFiles(fs opctx.FS, root string) (map[string]fileEntry, error) { +func collectFiles(fs opctx.FS, root string, ignoredPaths map[string]bool) (map[string]fileEntry, error) { result := make(map[string]fileEntry) walkErr := afero.Walk(fs, root, func(fullPath string, info os.FileInfo, err error) error { @@ -130,20 +135,26 @@ func collectFiles(fs opctx.FS, root string) (map[string]fileEntry, error) { return fmt.Errorf("error walking %#q:\n%w", fullPath, err) } - mode := info.Mode() + relPath, relErr := filepath.Rel(root, fullPath) + if relErr != nil { + return fmt.Errorf("failed to compute relative path for %#q from %#q:\n%w", fullPath, root, relErr) + } + + if ignoredPaths[relPath] { + if info.IsDir() { + return filepath.SkipDir + } - // Skip directories — we only collect leaf entries. Symlinks to directories - // are not reported as IsDir() by afero.Walk (which uses LstatIfPossible), - // so they correctly fall through to the symlink case below. - if info.IsDir() { return nil } - relPath, relErr := filepath.Rel(root, fullPath) - if relErr != nil { - return fmt.Errorf("failed to compute relative path for %#q from %#q:\n%w", fullPath, root, relErr) + // Walk reports symlinks to directories as symlinks, so only real directories are skipped. + if info.IsDir() { + return nil } + mode := info.Mode() + switch { case mode&os.ModeSymlink != 0: target, linkErr := readSymlinkTarget(fs, fullPath) diff --git a/internal/utils/dirdiff/dirdiff_test.go b/internal/utils/dirdiff/dirdiff_test.go index 4974eaa81..adeffcd15 100644 --- a/internal/utils/dirdiff/dirdiff_test.go +++ b/internal/utils/dirdiff/dirdiff_test.go @@ -15,6 +15,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -32,6 +33,95 @@ func TestDiffDirs_IdenticalDirs(t *testing.T) { assert.Empty(t, result.String()) } +func TestDiffDirs_IgnoredPaths(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name string + gitPath string + }{ + {name: "directory", gitPath: ".git/HEAD"}, + {name: "file", gitPath: ".git"}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + baseFS := afero.NewMemMapFs() + + for _, root := range []string{"/a", "/b"} { + for _, path := range []string{testCase.gitPath, ".gitignore", "nested/.git/HEAD"} { + require.NoError(t, fileutils.WriteFile(baseFS, filepath.Join(root, path), + []byte(root), fileperms.PublicFile)) + } + } + + testFS := unreadablePathFS{Fs: baseFS, blockedPath: "/a/.git"} + result, err := dirdiff.DiffDirs(testFS, "/a", "/b", dirdiff.WithIgnoredPaths(".git")) + require.NoError(t, err) + require.Len(t, result.Files, 2) + assert.Equal(t, ".gitignore", result.Files[0].Path) + assert.Equal(t, filepath.Join("nested", ".git", "HEAD"), result.Files[1].Path) + + _, err = dirdiff.DiffDirs(testFS, "/a", "/b") + require.ErrorIs(t, err, os.ErrPermission) + }) + } +} + +type unreadablePathFS struct { + afero.Fs + blockedPath string +} + +func (fs unreadablePathFS) Open(name string) (afero.File, error) { + if name == fs.blockedPath || strings.HasPrefix(name, fs.blockedPath+string(filepath.Separator)) { + return nil, os.ErrPermission + } + + return fs.Fs.Open(name) +} + +func TestDiffDirs_IgnoredOriginalPaths(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + name string + inModified bool + }{ + {name: "original only"}, + {name: "modified entry is still compared", inModified: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + baseFS := afero.NewMemMapFs() + for _, root := range []string{"/a", "/b"} { + require.NoError(t, fileutils.WriteFile(baseFS, filepath.Join(root, ".git/HEAD"), + []byte(root), fileperms.PublicFile)) + } + + require.NoError(t, fileutils.WriteFile(baseFS, "/a/marker", []byte("original"), fileperms.PublicFile)) + + if testCase.inModified { + require.NoError(t, fileutils.WriteFile(baseFS, "/b/marker", []byte("modified"), fileperms.PublicFile)) + } + + fs := unreadablePathFS{Fs: baseFS, blockedPath: "/a/marker"} + result, err := dirdiff.DiffDirs(fs, "/a", "/b", + dirdiff.WithIgnoredPaths(".git"), dirdiff.WithIgnoredOriginalPaths("marker")) + require.NoError(t, err) + + if testCase.inModified { + require.Len(t, result.Files, 1) + assert.Equal(t, "marker", result.Files[0].Path) + assert.Equal(t, dirdiff.FileStatusAdded, result.Files[0].Status) + } else { + assert.Empty(t, result.Files) + } + }) + } +} + func TestDiffDirs_ModifiedFile(t *testing.T) { ctx := testctx.NewCtx() testFS := ctx.FS() diff --git a/internal/utils/dirdiff/types.go b/internal/utils/dirdiff/types.go index 64a62f9b3..4ae547c96 100644 --- a/internal/utils/dirdiff/types.go +++ b/internal/utils/dirdiff/types.go @@ -3,6 +3,8 @@ package dirdiff +import "path/filepath" + // FileStatus describes whether a file was added, removed, modified, or changed type // relative to the original (pre-overlay) directory. type FileStatus string @@ -58,8 +60,10 @@ const ( // diffConfig holds the resolved configuration for a [DiffDirs] invocation. type diffConfig struct { - contextLines int - maxBinaryScanSize int + contextLines int + maxBinaryScanSize int + ignoredPaths map[string]bool + ignoredOriginalPaths map[string]bool } func newDiffConfig(opts []DiffOption) *diffConfig { @@ -93,6 +97,28 @@ func WithMaxBinaryScanBytes(n int) DiffOption { } } +// WithIgnoredPaths excludes paths relative to each compared root. Ignored +// directories are not traversed. Each call replaces the previous ignored paths. +func WithIgnoredPaths(paths ...string) DiffOption { + return func(cfg *diffConfig) { + cfg.ignoredPaths = make(map[string]bool, len(paths)) + for _, path := range paths { + cfg.ignoredPaths[filepath.Clean(path)] = true + } + } +} + +// WithIgnoredOriginalPaths additionally excludes root-relative paths only from +// the original (dirA) tree. Matching entries in the modified tree are still compared. +func WithIgnoredOriginalPaths(paths ...string) DiffOption { + return func(cfg *diffConfig) { + cfg.ignoredOriginalPaths = make(map[string]bool, len(paths)) + for _, path := range paths { + cfg.ignoredOriginalPaths[filepath.Clean(path)] = true + } + } +} + // fileKind categorizes directory entries for diffing purposes. type fileKind int diff --git a/internal/utils/git/git.go b/internal/utils/git/git.go index 7d5c780ea..b42998c0a 100644 --- a/internal/utils/git/git.go +++ b/internal/utils/git/git.go @@ -204,13 +204,21 @@ func resolveCloneOptions(options []GitOptions) cloneOptions { return resolved } -// WithGitBranch returns a [GitOptions] that specifies the branch to clone. +// WithGitBranch returns a [GitOptions] that selects the initial branch without +// excluding other branches' commits. func WithGitBranch(branch string) GitOptions { return func(opts *cloneOptions) { opts.args = append(opts.args, "--branch", branch) } } +// WithSingleBranch returns a [GitOptions] that fetches only the selected branch. +func WithSingleBranch() GitOptions { + return func(opts *cloneOptions) { + opts.args = append(opts.args, "--single-branch") + } +} + // WithQuiet returns a [GitOptions] that suppresses event emission during // the clone. Use this for internal operations (e.g., identity resolution) // that run concurrently and would produce misleading nested log output. diff --git a/internal/utils/git/git_test.go b/internal/utils/git/git_test.go index 08ded6879..8f49dad3f 100644 --- a/internal/utils/git/git_test.go +++ b/internal/utils/git/git_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx/opctx_test" "github.com/microsoft/azure-linux-dev-tools/internal/utils/externalcmd" "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" @@ -30,7 +31,6 @@ const ( testRepoBranch = "test" testRepoSubDir = "repo" testRepoReadmeFile = "README" - testRepoContribFile = "CONTRIBUTING.md" testGitDir = ".git" nonExistentRepoURL = "https://github.com/nonexistent/repository-does-not-exist-12345.git" errMsgEmptyURL = "repository URL cannot be empty" @@ -115,19 +115,22 @@ func TestClone(t *testing.T) { } func TestCloneWithBranch(t *testing.T) { - ctrl := gomock.NewController(t) - - cmdFactory, err := externalcmd.NewCmdFactory( - opctx_test.NewNoOpMockDryRunnable(ctrl), - opctx_test.NewNoOpMockEventListener(ctrl), - ) + env := testutils.NewTestEnv(t) + provider, err := git.NewGitProviderImpl(env.EventListener, env.CmdFactory) require.NoError(t, err) - provider, err := git.NewGitProviderImpl(opctx_test.NewNoOpMockEventListener(ctrl), cmdFactory) - require.NoError(t, err) + const destDir = "/working/repo" + + called := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + called = true - tempDir := t.TempDir() - destDir := filepath.Join(tempDir, testRepoSubDir) + assert.Equal(t, []string{ + "git", "clone", "--branch", testRepoBranch, testRepoURL, destDir, + }, cmd.Args) + + return nil + } err = provider.Clone( context.Background(), @@ -137,9 +140,28 @@ func TestCloneWithBranch(t *testing.T) { ) require.NoError(t, err) - assert.DirExists(t, destDir) - assert.DirExists(t, filepath.Join(destDir, testGitDir)) - assert.FileExists(t, filepath.Join(destDir, testRepoContribFile)) + assert.True(t, called) +} + +func TestCloneWithSingleBranch(t *testing.T) { + env := testutils.NewTestEnv(t) + provider, err := git.NewGitProviderImpl(env.EventListener, env.CmdFactory) + require.NoError(t, err) + + called := false + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + called = true + + assert.Equal(t, []string{ + "git", "clone", "--branch", testRepoBranch, "--single-branch", testRepoURL, "/working/repo", + }, cmd.Args) + + return nil + } + + require.NoError(t, provider.Clone(t.Context(), testRepoURL, "/working/repo", + git.WithGitBranch(testRepoBranch), git.WithSingleBranch())) + assert.True(t, called) } func TestCloneNonExistentRepo(t *testing.T) { diff --git a/magefiles/magefile.go b/magefiles/magefile.go index feddd9d98..d7a3a4b71 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -48,8 +48,10 @@ func init() { } } -// All runs the tests, builds the code, and runs the default checks. +// All builds the code, runs tests and default checks in parallel, then generates docs. func All() error { - mg.SerialDeps(magebuild.Build, magebuild.Unit, mg.F(magecheckfix.Check, magecheckfix.TargetDefault), magescenario.Scenario, magebuild.Docs) + mg.SerialDeps(magebuild.Build) + mg.Deps(magebuild.Unit, mg.F(magecheckfix.Check, magecheckfix.TargetDefault), magescenario.Scenario) + mg.SerialDeps(magebuild.Docs) return nil } diff --git a/magefiles/mageutil/mageutil.go b/magefiles/mageutil/mageutil.go index 2e1cddda4..28503f4ff 100644 --- a/magefiles/mageutil/mageutil.go +++ b/magefiles/mageutil/mageutil.go @@ -257,45 +257,13 @@ func createDir(dir string) error { // GetToolAbsPath returns the command to run a go tool that is managed within // one of the tools-focused modules in this repository. func GetToolAbsPath(toolName string) (toolPath string, err error) { - var origWd string - - // Save the current working directory so we can restore it later. - origWd, err = os.Getwd() - if err != nil { - return "", fmt.Errorf("%w: could not get current working directory: %w", ErrCmdLookup, err) - } - - // Restore the original working directory before returning. - defer func() { - chdirErr := os.Chdir(origWd) - if chdirErr != nil { - if err == nil { - err = fmt.Errorf("%w: could not restore original working directory %q: %w", - ErrCmdLookup, origWd, chdirErr) - } else { - err = fmt.Errorf("%w: could not restore original working directory %q: %w; original error: %w", - ErrCmdLookup, origWd, chdirErr, err) - } - } - }() + toolWd := filepath.Join(AzldevProjectDir(), "tools", toolName) // NOTE: We've found that we typically get a temporary-and-no-longer-resolvable path back // the first time we invoke "go tool". Thereafter, it's in the cache. We call a second // time for the *good* path. for range 2 { - var err error - - toolWd := filepath.Join(origWd, "tools", toolName) - - // NOTE: To avoid errors with "go" not being willing to handle the -modfile flag in case - // a go workspace (go.work) is present, we chdir to the directory where we expect to find - // the go.mod. - err = os.Chdir(filepath.Join(origWd, "tools", toolName)) - if err != nil { - return "", fmt.Errorf("%w: could not change directory to %q: %w", ErrCmdLookup, toolWd, err) - } - - toolPath, err = sh.Output(mg.GoCmd(), "tool", "-n", toolName) + toolPath, err = sh.Output(mg.GoCmd(), "-C", toolWd, "tool", "-n", toolName) if err != nil { return "", fmt.Errorf("%w: could not find the executable path for tool %q using cwd %q: %w", ErrCmdLookup, toolName, toolWd, err) diff --git a/pkg/app/azldev_cli/azldev.go b/pkg/app/azldev_cli/azldev.go index 786e5df6b..1aa0b5633 100644 --- a/pkg/app/azldev_cli/azldev.go +++ b/pkg/app/azldev_cli/azldev.go @@ -39,20 +39,18 @@ func Main() { } // InstantiateApp constructs a new instance of the azldev CLI application with -// all subcommands registered for azldev's default (lock file) mode. +// all subcommands registered. func InstantiateApp() *azldev.App { return InstantiateAppForArgs(nil) } // InstantiateAppForArgs constructs a new instance of the azldev CLI application -// with the subcommands registered for the mode selected by args. Global flags -// that select a mode (such as '--without-lockfile') are hand-parsed from args -// before registration, because the registered command set differs by mode. +// with configuration flags pre-parsed from args and all subcommands registered. func InstantiateAppForArgs(args []string) *azldev.App { // Instantiate the main CLI application. app := azldev.NewApp(azldev.DefaultFileSystemFactory(), azldev.DefaultOSEnvFactory()) - // Resolve the global flags that command registration depends on. + // Pre-parse flags needed for configuration loading. app.PreParseGlobalFlags(args) // Give top level command packages an opportunity to register their commands (or in some cases, diff --git a/scenario/__snapshots__/TestMCPServerMode_1.snap.json b/scenario/__snapshots__/TestMCPServerMode_1.snap.json index 663b83ff7..5119f0cfd 100755 --- a/scenario/__snapshots__/TestMCPServerMode_1.snap.json +++ b/scenario/__snapshots__/TestMCPServerMode_1.snap.json @@ -105,11 +105,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [ @@ -216,11 +211,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -325,11 +315,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -424,11 +409,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -543,11 +523,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -625,11 +600,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -703,11 +673,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -786,11 +751,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -879,11 +839,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [ @@ -959,11 +914,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], @@ -1056,11 +1006,6 @@ "default": false, "description": "enable verbose output", "type": "boolean" - }, - "without-lockfile": { - "default": false, - "description": "preview: track resolved upstream commits in generated config instead of lock files", - "type": "boolean" } }, "required": [], diff --git a/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap index 6832b09a4..48c9a1b58 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_--bogus-flag_stderr_1.snap @@ -31,7 +31,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_--help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 9cf47bd35..795c1ea39 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -345,7 +345,7 @@ "skip-file-filter": { "type": "boolean", "title": "Skip file filter", - "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + "description": "Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap b/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap index 6832b09a4..48c9a1b58 100755 --- a/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--bogus-flag_stderr_1.snap @@ -31,7 +31,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap index 4a47c31e2..3dfbd2bb4 100755 --- a/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_--help_with_color_stdout_1.snap @@ -38,7 +38,6 @@ Meta commands: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 9cf47bd35..795c1ea39 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -345,7 +345,7 @@ "skip-file-filter": { "type": "boolean", "title": "Skip file filter", - "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + "description": "Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git." } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap index e15aef19c..0b77a7417 100755 --- a/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_help_stdout_1.snap @@ -38,7 +38,6 @@ Flags: -q, --quiet only enable minimal output -v, --verbose enable verbose output --version version for azldev - --without-lockfile preview: track resolved upstream commits in generated config instead of lock files Use "azldev [command] --help" for more information about a command. Use "azldev advanced --help" for additional tools (mock, mcp, wget). diff --git a/scenario/without_lockfile_test.go b/scenario/without_lockfile_test.go index 23ccd4cbc..b42a85838 100644 --- a/scenario/without_lockfile_test.go +++ b/scenario/without_lockfile_test.go @@ -22,13 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -// These tests cover the opt-in lock-file-free mode selected by the global -// '--without-lockfile' flag, and assert that omitting the flag keeps azldev's -// default lock-file behavior. - -// TestWithoutLockfile_GlobalFlagIsDocumented verifies that the preview flag is part -// of the CLI surface and defaults to off. -func TestWithoutLockfile_GlobalFlagIsDocumented(t *testing.T) { +func TestLocklessComponents_NoGlobalFlag(t *testing.T) { t.Parallel() if testing.Short() { @@ -38,69 +32,25 @@ func TestWithoutLockfile_GlobalFlagIsDocumented(t *testing.T) { results, err := cmdtest.NewScenarioTest("--help").Locally().Run(t) require.NoError(t, err) require.Zero(t, results.ExitCode) - assert.Contains(t, results.Stdout, "--without-lockfile") + assert.NotContains(t, results.Stdout, "--without-lockfile") } -// TestWithoutLockfile_ComponentCommandsByMode verifies that the component command -// set matches the selected mode: the lock-file commands by default, and the -// refresh-upstream-commit command when the preview flag is passed. -func TestWithoutLockfile_ComponentCommandsByMode(t *testing.T) { +func TestLocklessComponents_BothWorkflowsAvailable(t *testing.T) { t.Parallel() if testing.Short() { t.Skip("skipping long test") } - testCases := []struct { - name string - args []string - expected []string - notExpected []string - }{ - { - name: "default mode", - args: []string{"component", "--help"}, - expected: []string{"update", "history", "query"}, - notExpected: []string{"refresh-upstream-commit"}, - }, - { - name: "explicitly disabled", - args: []string{"--without-lockfile=false", "component", "--help"}, - expected: []string{"update", "history", "query"}, - notExpected: []string{"refresh-upstream-commit"}, - }, - { - name: "lock-file-free mode", - args: []string{"--without-lockfile", "component", "--help"}, - expected: []string{"refresh-upstream-commit"}, - // The lock-file commands remain registered as hidden no-ops, so they - // must not be advertised in help. - notExpected: []string{"Refresh component lock files"}, - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - t.Parallel() - - results, err := cmdtest.NewScenarioTest(testCase.args...).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, results.ExitCode) - - for _, expected := range testCase.expected { - assert.Contains(t, results.Stdout, expected) - } - - for _, notExpected := range testCase.notExpected { - assert.NotContains(t, results.Stdout, notExpected) - } - }) + results, err := cmdtest.NewScenarioTest("component", "--help").Locally().Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode) + for _, name := range []string{"update", "history", "query", "refresh-upstream-commit"} { + assert.Contains(t, results.Stdout, name) } } -// TestWithoutLockfile_BuildFlagsByMode verifies that the lock-file-only component -// flags are registered only in the default mode. -func TestWithoutLockfile_BuildFlagsByMode(t *testing.T) { +func TestLocklessComponents_LockValidationFlagAvailable(t *testing.T) { t.Parallel() if testing.Short() { @@ -112,34 +62,9 @@ func TestWithoutLockfile_BuildFlagsByMode(t *testing.T) { require.Zero(t, defaultResults.ExitCode) assert.Contains(t, defaultResults.Stdout, "--skip-lock-validation") - previewResults, err := cmdtest.NewScenarioTest( - "--without-lockfile", "component", "build", "--help", - ).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, previewResults.ExitCode) - assert.NotContains(t, previewResults.Stdout, "--skip-lock-validation") -} - -// TestWithoutLockfile_LegacyCommandIsNoOp verifies that a lock-file command invoked -// in lock-file-free mode reports that it does nothing instead of failing. -func TestWithoutLockfile_LegacyCommandIsNoOp(t *testing.T) { - t.Parallel() - - if testing.Short() { - t.Skip("skipping long test") - } - - results, err := cmdtest.NewScenarioTest( - "--without-lockfile", "component", "update", "--all-components", - ).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, results.ExitCode) - assert.Contains(t, results.Stdout+results.Stderr, "no longer does anything") } -// TestWithoutLockfile_AgentSkillsByMode verifies that the emitted agent skills -// describe the workflow of the selected mode. -func TestWithoutLockfile_AgentSkillsByMode(t *testing.T) { +func TestLocklessComponents_AgentSkillsCoverBothWorkflows(t *testing.T) { t.Parallel() if testing.Short() { @@ -150,29 +75,11 @@ func TestWithoutLockfile_AgentSkillsByMode(t *testing.T) { require.NoError(t, err) require.Zero(t, defaultResults.ExitCode) assert.Contains(t, defaultResults.Stdout, "azldev-update-component") - assert.NotContains(t, defaultResults.Stdout, "azldev-refresh-upstream-commit") + assert.Contains(t, defaultResults.Stdout, "azldev-refresh-upstream-commit") - previewResults, err := cmdtest.NewScenarioTest( - "--without-lockfile", "docs", "agent", "show", - ).Locally().Run(t) - require.NoError(t, err) - require.Zero(t, previewResults.ExitCode) - assert.Contains(t, previewResults.Stdout, "azldev-refresh-upstream-commit") - assert.NotContains(t, previewResults.Stdout, "azldev-update-component") } -// TestWithoutLockfile_ComponentChanged compares components across two commits in -// lock-file-free mode, where change detection compares the project configuration -// resolved at each ref instead of stored lock files. -// -// Flow: -// 1. Create a project with two local components (curl, bash). -// 2. Commit the project as the baseline. -// 3. In a second commit, change only curl's spec content. -// 4. Run 'azldev --without-lockfile component changed' between the two commits. -// 5. Assert curl is "changed" (its spec directory contents differ) and bash is -// "unchanged", with no lock files anywhere in the project. -func TestWithoutLockfile_ComponentChanged(t *testing.T) { +func TestLocklessComponents_ComponentChanged(t *testing.T) { t.Parallel() if testing.Short() { @@ -216,6 +123,8 @@ func TestWithoutLockfile_ComponentChanged(t *testing.T) { project.Serialize(t, projectDir) patchProjectForLocal(t, projectDir) + require.NoError(t, os.WriteFile(filepath.Join(projectDir, "lockless-components.txt"), + []byte("curl\n"), fileperms.PublicFile)) gitInDir(t, projectDir, "init") gitInDir(t, projectDir, "config", "user.email", "test@test.com") @@ -238,10 +147,11 @@ func TestWithoutLockfile_ComponentChanged(t *testing.T) { gitInDir(t, projectDir, "add", filepath.Join("specs", "curl", "curl.spec")) gitInDir(t, projectDir, "-c", "commit.gpgsign=false", "commit", "-m", "change curl") - cmd := exec.CommandContext(t.Context(), - azldevBin, "--without-lockfile", "-C", projectDir, "--no-default-config", + args := []string{ + "-C", projectDir, "--no-default-config", "component", "changed", "--from", fromRef, "-a", "--include-unchanged", "-q", "-O", "json", - ) + } + cmd := exec.CommandContext(t.Context(), azldevBin, args...) out, err := cmd.CombinedOutput() require.NoError(t, err, "azldev failed: %s", string(out)) @@ -262,19 +172,17 @@ func TestWithoutLockfile_ComponentChanged(t *testing.T) { require.True(t, ok, "bash should be in results (--all-components)") assert.Equal(t, "unchanged", bashResult.ChangeType, "bash is untouched") - // No lock files are consulted or created in this mode. + // Change detection does not create lock files. entries, err := os.ReadDir(projectDir) require.NoError(t, err) for _, entry := range entries { - assert.NotEqual(t, "locks", entry.Name(), "lock-file-free mode must not create a lock directory") + assert.NotEqual(t, "locks", entry.Name(), "change detection must not create a lock directory") assert.False(t, strings.HasSuffix(entry.Name(), ".lock")) } } -// TestWithoutLockfile_MCPToolsByMode verifies that the MCP tool surface follows the -// selected mode's command set. -func TestWithoutLockfile_MCPToolsByMode(t *testing.T) { +func TestLocklessComponents_MCPTools(t *testing.T) { t.Parallel() if testing.Short() { @@ -289,10 +197,5 @@ func TestWithoutLockfile_MCPToolsByMode(t *testing.T) { require.Zero(t, defaultResults.ExitCode) assert.Contains(t, defaultResults.Stdout, `"component-history"`) - previewResults, err := cmdtest.NewScenarioTest("--without-lockfile", "advanced", "mcp"). - Locally().WithStdin(strings.NewReader(listToolsRequest)).Run(t) - require.NoError(t, err) - require.Zero(t, previewResults.ExitCode) - assert.NotContains(t, previewResults.Stdout, `"component-history"`) - assert.Contains(t, previewResults.Stdout, `"component-changed"`) + assert.Contains(t, defaultResults.Stdout, `"component-changed"`) } diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 9cf47bd35..795c1ea39 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -345,7 +345,7 @@ "skip-file-filter": { "type": "boolean", "title": "Skip file filter", - "description": "Disable post-render file filtering for specs with unexpandable macros in Source/Patch tags" + "description": "Disable post-render file filtering for locked components with unexpandable macros in Source/Patch tags. Lockless rendering always preserves all prepared files except .git." } }, "additionalProperties": false,