diff --git a/Cargo.toml b/Cargo.toml index 03605b0..f4f3a4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,7 +123,7 @@ unsafe_code = "forbid" missing_docs = { level = "deny", priority = 0 } dead_code = { level = "deny", priority = 0 } unreachable_pub = { level = "deny", priority = 0 } -unexpected_cfgs = { level = "deny", priority = 0, check-cfg = [ 'cfg(docsrs)', 'cfg(la_stack_pre_rational_input_api)' ] } +unexpected_cfgs = { level = "deny", priority = 0, check-cfg = [ 'cfg(docsrs)' ] } [workspace.lints.rustdoc] bare_urls = "deny" diff --git a/benches/common/exact.rs b/benches/common/exact.rs index f18a5ed..7f014c8 100644 --- a/benches/common/exact.rs +++ b/benches/common/exact.rs @@ -598,10 +598,6 @@ pub fn validate_exact_fixture(input: ExactInput) -> Validated .matrix .solve_exact(input.rhs) .or_abort("exact solve oracle check"); - // Historical releases return the rational array directly instead of a wrapper. - #[cfg(la_stack_pre_rational_input_api)] - let solution = &solution; - #[cfg(not(la_stack_pre_rational_input_api))] let solution = solution.as_array(); assert_exact_residual(&input, solution); diff --git a/benches/common/vs_linalg.rs b/benches/common/vs_linalg.rs index 7bf4ca7..697d115 100644 --- a/benches/common/vs_linalg.rs +++ b/benches/common/vs_linalg.rs @@ -8,58 +8,10 @@ use faer::linalg::solvers::{Ldlt as FaerLdlt, PartialPivLu}; use faer::perm::PermRef; use nalgebra::SMatrix; -use la_stack::{LaError, Matrix, Tolerance, Vector}; +use la_stack::{Matrix, Tolerance, Vector}; use crate::bench_utils::OrAbort; -/// Evaluate the squared norm through the current public API. -/// -/// # Errors -/// -/// Returns the library's typed error if squared-norm accumulation overflows. -#[cfg(not(la_stack_pre_rational_input_api))] -#[inline] -pub const fn la_stack_norm_squared(vector: &Vector) -> Result { - vector.norm_squared() -} - -/// Evaluate the same squared norm through the pre-v0.4.6 method name. -/// -/// This adapter preserves historical comparisons without adding library aliases. -/// -/// # Errors -/// -/// Returns the selected revision's typed error if squared-norm accumulation overflows. -#[cfg(la_stack_pre_rational_input_api)] -#[inline] -pub const fn la_stack_norm_squared(vector: &Vector) -> Result { - vector.norm2_sq() -} - -/// Evaluate the matrix infinity norm through the current public API. -/// -/// # Errors -/// -/// Returns the library's typed error if an absolute row sum overflows. -#[cfg(not(la_stack_pre_rational_input_api))] -#[inline] -pub const fn la_stack_norm_inf(matrix: &Matrix) -> Result { - matrix.norm_inf() -} - -/// Evaluate the same matrix infinity norm through the pre-v0.4.6 method name. -/// -/// This adapter preserves historical comparisons without adding library aliases. -/// -/// # Errors -/// -/// Returns the selected revision's typed error if an absolute row sum overflows. -#[cfg(la_stack_pre_rational_input_api)] -#[inline] -pub const fn la_stack_norm_inf(matrix: &Matrix) -> Result { - matrix.inf_norm() -} - /// Return `det(P)` for faer's permutation representation. /// /// Sign(det(P)) is +1 for even permutations and -1 for odd. Parity is computed diff --git a/benches/comparison/tests/vs_linalg_inputs.rs b/benches/comparison/tests/vs_linalg_inputs.rs index d624cfe..bf50b4e 100644 --- a/benches/comparison/tests/vs_linalg_inputs.rs +++ b/benches/comparison/tests/vs_linalg_inputs.rs @@ -25,11 +25,10 @@ use vs_linalg_common::{ LuSolveScenario, make_balanced_dynamic_range_rows, validated_lu_solve_input, }; use vs_linalg_common::{ - PreparedFaerLuDet, faer_det_from_ldlt, faer_perm_sign, la_stack_norm_inf, - la_stack_norm_squared, make_ill_conditioned_matrix_rows, make_matrix_rows, - make_pivoting_matrix_rows, make_vector_array, matrix_entry, nalgebra_inf_norm, vector_entry, + PreparedFaerLuDet, faer_det_from_ldlt, faer_perm_sign, make_ill_conditioned_matrix_rows, + make_matrix_rows, make_pivoting_matrix_rows, make_vector_array, matrix_entry, + nalgebra_inf_norm, vector_entry, }; -#[cfg(not(la_stack_pre_rational_input_api))] use vs_linalg_common::{ delaunay_scaled_norm, iterative_hypot, make_norm_descending_array, make_norm_repeated_scale_array, make_norm_sparse_array, make_norm_wide_dynamic_range_array, @@ -221,44 +220,41 @@ fn assert_vector_operation_agreement() { assert_abs_diff_eq!(nv1.dot(&nv2), expected_dot, epsilon = 0.0); assert_abs_diff_eq!(fa_dot, expected_dot, epsilon = 0.0); - let la_norm_squared = la_stack_norm_squared(&v1) + let la_norm_squared = v1 + .norm_squared() .unwrap_or_else(|err| panic!("la_stack norm_squared failed: {err}")); assert_close("nalgebra_norm_squared", nv1.norm_squared(), la_norm_squared); let fa_norm_squared = fv1.as_mat_ref().squared_norm_l2(); assert_close("faer_norm2_sq", fa_norm_squared, la_norm_squared); - #[cfg(not(la_stack_pre_rational_input_api))] - { - let la_norm = v1 + let la_norm = v1 + .norm() + .unwrap_or_else(|err| panic!("la_stack norm failed: {err}")); + assert_close( + "iterative_f64_hypot", + iterative_hypot(v1.as_array()), + la_norm, + ); + assert_close( + "delaunay_scaled_norm", + delaunay_scaled_norm(v1.as_array()), + la_norm, + ); + assert_close("nalgebra_norm", nv1.norm(), la_norm); + assert_close("faer_norm_l2", fv1.as_mat_ref().norm_l2(), la_norm); + + for (scenario, values) in norm_scenarios::() { + let vector = Vector::::try_new(values).unwrap_or_else(|err| { + panic!("la_stack {scenario} norm scenario construction failed: {err}") + }); + let expected = vector .norm() - .unwrap_or_else(|err| panic!("la_stack norm failed: {err}")); - assert_close( - "iterative_f64_hypot", - iterative_hypot(v1.as_array()), - la_norm, - ); - assert_close( - "delaunay_scaled_norm", - delaunay_scaled_norm(v1.as_array()), - la_norm, - ); - assert_close("nalgebra_norm", nv1.norm(), la_norm); - assert_close("faer_norm_l2", fv1.as_mat_ref().norm_l2(), la_norm); - - for (scenario, values) in norm_scenarios::() { - let vector = Vector::::try_new(values).unwrap_or_else(|err| { - panic!("la_stack {scenario} norm scenario construction failed: {err}") - }); - let expected = vector - .norm() - .unwrap_or_else(|err| panic!("la_stack {scenario} norm failed: {err}")); - assert_close(scenario, iterative_hypot(&values), expected); - assert_close(scenario, delaunay_scaled_norm(&values), expected); - } + .unwrap_or_else(|err| panic!("la_stack {scenario} norm failed: {err}")); + assert_close(scenario, iterative_hypot(&values), expected); + assert_close(scenario, delaunay_scaled_norm(&values), expected); } } -#[cfg(not(la_stack_pre_rational_input_api))] #[test] fn norm_scenario_inputs_cover_distinct_branch_and_range_profiles() { let descending = make_norm_descending_array::<8>(); @@ -430,8 +426,9 @@ fn assert_matrix_norm_inf_agreement() { let na = SMatrix::::from_fn(matrix_entry::); let fa = Mat::::from_fn(D, D, matrix_entry::); - let la_norm = - la_stack_norm_inf(&a).unwrap_or_else(|err| panic!("la_stack norm_inf failed: {err}")); + let la_norm = a + .norm_inf() + .unwrap_or_else(|err| panic!("la_stack norm_inf failed: {err}")); assert_close("nalgebra_inf_norm", nalgebra_inf_norm(&na), la_norm); let mut fa_norm = 0.0; for r in 0..D { diff --git a/benches/exact.rs b/benches/exact.rs index 0deb2da..dd710bc 100644 --- a/benches/exact.rs +++ b/benches/exact.rs @@ -35,7 +35,6 @@ use std::hint::black_box; use criterion::{BenchmarkGroup, Criterion, Throughput, measurement::WallTime}; -#[cfg(not(la_stack_pre_rational_input_api))] use la_stack::ExactF64Conversion; use la_stack::{Matrix, Vector}; @@ -44,11 +43,9 @@ mod bench_utils; #[path = "common/exact.rs"] pub mod exact_bench; -#[cfg(not(la_stack_pre_rational_input_api))] #[path = "common/rational.rs"] pub mod rational_bench; -#[cfg(not(la_stack_pre_rational_input_api))] #[path = "common/exact_diagnostics.rs"] pub mod exact_diagnostics; @@ -58,12 +55,10 @@ use exact_bench::{ large_entries_3x3_input, make_matrix_rows, make_random_input_corpus, make_vector_array, near_singular_3x3_input, validate_exact_fixture, validate_f64_determinant_benchmarks, }; -#[cfg(not(la_stack_pre_rational_input_api))] use exact_diagnostics::{ ConversionKind, Det4Kind, EXPONENT_SPREADS, ExponentAxis, canonical_conversion_input, exact_det4_input, exponent_spread_input, }; -#[cfg(not(la_stack_pre_rational_input_api))] use rational_bench::{ RationalInputKind, rational_determinant_gaussian, rational_input, rational_solve_gaussian, }; @@ -119,7 +114,6 @@ const CORPUS_AND_EXTREME_OPERATIONS: &[ExactOperation] = &[ /// Both algorithms measure a complete operation on borrowed, accepted input. /// The consuming Gaussian references therefore make their required working /// copies inside the timed closure, just as row clearing builds its workspace. -#[cfg(not(la_stack_pre_rational_input_api))] fn bench_rational_input(criterion: &mut Criterion, kind: RationalInputKind) { let input = rational_input::(kind); let group_name = match kind { @@ -379,7 +373,6 @@ macro_rules! gen_random_corpus_benches_for_dim { }}; } -#[cfg(not(la_stack_pre_rational_input_api))] fn bench_canonical_conversion(c: &mut Criterion) { for kind in ConversionKind::ALL { let input = canonical_conversion_input::(kind); @@ -394,7 +387,6 @@ fn bench_canonical_conversion(c: &mut Criterion) { } } -#[cfg(not(la_stack_pre_rational_input_api))] fn bench_det4_diagnostics(c: &mut Criterion) { for kind in Det4Kind::ALL { let input = exact_det4_input(kind); @@ -415,7 +407,6 @@ fn bench_det4_diagnostics(c: &mut Criterion) { } } -#[cfg(not(la_stack_pre_rational_input_api))] fn bench_exponent_spreads(c: &mut Criterion, axis: ExponentAxis) { for spread in EXPONENT_SPREADS { let input = exponent_spread_input::(spread, axis); @@ -455,7 +446,6 @@ fn main() { gen_random_corpus_benches_for_dim!(&mut c, 5); } - #[cfg(not(la_stack_pre_rational_input_api))] { bench_canonical_conversion::<2>(&mut c); bench_canonical_conversion::<3>(&mut c); diff --git a/benches/vs_linalg.rs b/benches/vs_linalg.rs index 6432150..f5f5593 100644 --- a/benches/vs_linalg.rs +++ b/benches/vs_linalg.rs @@ -29,14 +29,12 @@ mod bench_utils; pub mod vs_linalg_common; use bench_utils::OrAbort; -#[cfg(not(la_stack_pre_rational_input_api))] use vs_linalg_common::norm_scenarios; use vs_linalg_common::{LuSolveScenario, validated_lu_solve_input}; use vs_linalg_common::{ PreparedFaerLuDet, delaunay_scaled_norm, faer_det_from_ldlt, iterative_hypot, - la_stack_norm_inf, la_stack_norm_squared, make_balanced_dynamic_range_rows, - make_ill_conditioned_matrix_rows, make_matrix_rows, make_pivoting_matrix_rows, - make_vector_array, matrix_entry, nalgebra_inf_norm, vector_entry, + make_balanced_dynamic_range_rows, make_ill_conditioned_matrix_rows, make_matrix_rows, + make_pivoting_matrix_rows, make_vector_array, matrix_entry, nalgebra_inf_norm, vector_entry, }; /// Build the deterministic la-stack matrix shared by a benchmark family. @@ -447,7 +445,9 @@ fn register_vector_benchmarks(group: &mut BenchmarkGroup<'_, Wal group.bench_function("la_stack_norm2_sq", |bencher| { bencher.iter(|| { - let result = la_stack_norm_squared(black_box(&v1)).or_abort("la_stack norm_squared"); + let result = black_box(&v1) + .norm_squared() + .or_abort("la_stack norm_squared"); black_box(result); }); }); @@ -467,15 +467,12 @@ fn register_vector_benchmarks(group: &mut BenchmarkGroup<'_, Wal }); }); - #[cfg(not(la_stack_pre_rational_input_api))] - { - group.bench_function("la_stack_norm2", |bencher| { - bencher.iter(|| { - let result = black_box(&v1).norm().or_abort("la_stack norm"); - black_box(result); - }); + group.bench_function("la_stack_norm2", |bencher| { + bencher.iter(|| { + let result = black_box(&v1).norm().or_abort("la_stack norm"); + black_box(result); }); - } + }); group.bench_function("iterative_f64_hypot", |bencher| { bencher.iter(|| { @@ -507,7 +504,6 @@ fn register_vector_benchmarks(group: &mut BenchmarkGroup<'_, Wal } /// Validate and register safe Euclidean-norm kernels across branch and range profiles. -#[cfg(not(la_stack_pre_rational_input_api))] fn register_norm_scenario_benchmarks(group: &mut BenchmarkGroup<'_, WallTime>) { for (scenario, values) in norm_scenarios::() { let vector = Vector::try_new(values).or_abort("norm scenario vector construction"); @@ -562,7 +558,7 @@ fn register_matrix_norm_benchmarks(group: &mut BenchmarkGroup<'_ group.bench_function("la_stack_inf_norm", |bencher| { bencher.iter(|| { - let result = la_stack_norm_inf(black_box(&a)).or_abort("la_stack norm_inf"); + let result = black_box(&a).norm_inf().or_abort("la_stack norm_inf"); black_box(result); }); }); @@ -746,7 +742,6 @@ macro_rules! define_vs_linalg_benches_for_dim { register_precomputed_lu_determinant_benchmarks::<$d>(&mut group); register_precomputed_ldlt_determinant_benchmarks::<$d>(&mut group); register_vector_benchmarks::<$d>(&mut group); - #[cfg(not(la_stack_pre_rational_input_api))] register_norm_scenario_benchmarks::<$d>(&mut group); register_matrix_norm_benchmarks::<$d>(&mut group); $( diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 0401b90..d9a41f1 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -14,7 +14,6 @@ the commands measure and where their outputs go. - [Iterate Against A Local Saved Baseline](#iterate-against-a-local-saved-baseline) - [Update The README nalgebra/faer Table](#update-the-readme-nalgebrafaer-table) - [Create The Release Performance Report](#create-the-release-performance-report) - - [Compare Published Release Artifacts](#compare-published-release-artifacts) - [Output Locations](#output-locations) - [`vs_linalg` Methodology](#vs_linalg-methodology) - [Exact-Arithmetic Notes](#exact-arithmetic-notes) @@ -40,7 +39,6 @@ next release; measured artifact links and historical reports retain their tags. | README table and plot | `just performance-release` then `just performance-readme` | | Release report | `just performance-release "$CURRENT_TAG" "$PREVIOUS_TAG"` | | Build docs from retained release inputs | `just performance-doc` | -| Published-asset comparison | `just performance-github-assets "$CURRENT_TAG" "$PREVIOUS_TAG"` | Rule of thumb: @@ -215,18 +213,18 @@ absent from the isolated current phase. The retained complete-run inputs and Pyt renderer provide the release report. Baseline-saving recipes also disable unused Criterion plots and HTML without changing sampling. -New release comparisons require v0.4.4 or newer on both sides. v0.4.3 and older -are unsupported: the workflow rejects them before benchmark setup. Historical -Markdown reports remain in the archive, but are not regenerated with the current -harness. There are no v0.4.3 API adapters or missing-row exemptions. +New measurements require v0.5.0 or newer on both sides. Earlier releases are +rejected before fetching tags, creating worktrees or discovering benchmarks. +The old API and root-test layout adapters are retired; both phases must contain +every selected report row, including rational-input and norm cases. +Comparison builds cap lint diagnostics at warning for both revisions because +the current manifest's lint policy may reject released source that predates a +lint, even though that source remains valid benchmark input. -v0.4.4 and v0.4.5 use adapters for renamed norm methods and the array return -type of `solve_exact`, and omit the later rational-input API. These adapters -preserve the same operations and mathematical checks without patching either -library implementation. Comparison builds cap lint diagnostics at warning for -both revisions because the current manifest's lint policy may reject historical -source that predates a lint, even though that source remains valid benchmark -input. +Historical reports, raw artifacts and their hashes remain unchanged. The +measurement floor does not prevent offline rendering of retained complete runs +that satisfy the modern scientific policy. Older records requiring retired API +adapters remain readable directly in their original formats. This command does not depend on existing local `target/criterion/` baselines. It is slower than reusing a saved baseline, but less sensitive to stale local @@ -378,8 +376,8 @@ The native Criterion inventory declares every expected semantic ID, including diagnostics outside the selected report. Each case requires 100 raw samples, both mean and median estimates, and complete 95% intervals in nanoseconds. The consumer declares canonical report rows and baseline-only nalgebra/faer -reference series. The pre-rational-input adapter marks unsupported baseline -rows explicitly; current-only measurements never imply cross-release speedups. +reference series. Both phases must include every selected la-stack row; +missing baseline rows fail validation. Scratch evidence uses `performance.run.json` and `performance.evidence.json` under `target/bench-reports/`. These preserve raw Criterion JSON, both source @@ -439,28 +437,10 @@ retained and reproducible but cannot become release documentation. Use promotion for presentation-only report corrections; changes to benchmark inputs, code, toolchains, or measurement configuration require a fresh local or release run. -### Compare Published Release Artifacts - -After releases are published, the GitHub Release benchmark workflow attaches a -compressed Criterion baseline artifact. To compare those stored artifacts -without running cargo locally, install the GitHub CLI (`gh`) and authenticate it -with access to the repository (`gh auth login` or an equivalent token). The -requirement applies even when both release tags are supplied explicitly because -the recipe still downloads their GitHub Release assets: - -```bash -just performance-github-assets "$CURRENT_TAG" "$PREVIOUS_TAG" -``` - -With no arguments, the recipe discovers the latest and previous stable -published GitHub releases. - -Published artifacts preserve each release's original benchmark harness. Their -historical timing environments may not have been recorded, so report provenance -labels those fields, including historical correctness gates, unavailable. This -read-only comparison does not claim common-harness measurement or permit release -promotion. Use a local -shared-harness workflow before attributing a difference solely to library code. +The historical `performance-github-assets` comparison command is retired. +Published Criterion archives remain available as original release artifacts; +their missing source, host and gate provenance is not reconstructed. Use +`performance-local` or `performance-release` for new checked comparisons. ## Output Locations @@ -471,10 +451,9 @@ shared-harness workflow before attributing a difference solely to library code. | `target/bench-reports/performance.run.json` | No | `performance-local`, `performance-release` | Complete native cases, samples, and named phase series. | | `target/bench-reports/performance.evidence.json` | No | `performance-local`, `performance-release` | Shared evidence envelope with source/harness and phase provenance. | | `target/bench-reports/performance-non-exact.*` | No | `performance-local-non-exact` | Narrowed non-exact report and retained peer-context comparison inputs. | -| `target/bench-reports/github-assets-performance.md` | No | `performance-github-assets` | Local report from published release artifacts. | | `docs/performance.md` | Yes | `performance-release`, `performance-doc` | Current shared full report; last legacy report until complete-run promotion. | | `docs/performance-runs/` | Yes | `performance-release`, `performance-doc` | Immutable complete runs, reports, index, and validated latest pointer. | -| `docs/performance/` | Yes | Frozen record/shared conversion | Unchanged historical CSV/JSON evidence and hashes. | +| `docs/performance/` | Yes | Frozen record | Unchanged historical CSV/JSON evidence and hashes. | | `docs/archive/performance/` | Yes | Frozen record | Preserved reports in the historical format. | | `docs/archive/performance/studies/` | Yes | Maintainer investigations | Completed optimization studies and decisions. | | `docs/assets/bench/` | Yes | `performance-readme` | README benchmark CSV/SVG assets and JSON provenance. | @@ -530,9 +509,8 @@ report also includes dot products and other kernels. The `norm2`, `norm2_sq`, and `inf_norm` benchmark IDs, including scenario suffixes, are retained for continuity with saved baselines. The current public methods are -`Vector::norm()`, `Vector::norm_squared()`, and `Matrix::norm_inf()`; the historical -benchmark adapters call `norm2_sq()` and `inf_norm()` only when building against -older library releases. +`Vector::norm()`, `Vector::norm_squared()`, and `Matrix::norm_inf()`, which the +benchmark harness calls directly. The `norm2` family also measures iterative `f64::hypot` and Delaunay's existing dimension-specialized scaled implementation as labeled reference kernels. These @@ -755,11 +733,12 @@ The original `rational_input_d{2..8}` groups are part of the canonical exact release signal. The wide-component diagnostics remain local Criterion evidence and are not included in the release report's group registry. -Releases produced with the rational-input harness include their Criterion point -estimates and confidence intervals. When the comparison baseline predates the -rational-input API, the report retains current-only rows with an explicit -coverage note and does not calculate a cross-release ratio; historical reports -whose shared harness predates these groups omit them on both sides. +New exact comparisons require these groups, with Criterion point estimates and +confidence intervals, in both baseline and current phases. Frozen historical +reports may retain current-only rows with a coverage note and no cross-release +ratio when their baseline predates the rational-input API. Historical reports +whose shared harness predates these groups omit them on both sides; these +historical omissions do not qualify a run for current complete-run publication. The proof-bearing fixture is therefore a prerequisite correctness gate, not a claim that every timed sample is revalidated. Criterion closures remain free of diff --git a/docs/RELEASING.md b/docs/RELEASING.md index a20e53a..a616d81 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -393,7 +393,8 @@ cargo check --locked --manifest-path "$CONSUMER/Cargo.toml" Use a disposable shell for this check. Preserve the output and record the tag, commit, benchmark run, publishing run, registry URL/checksum and consumer results -in #259. Keep that issue open until the approved v0.5.0 publication is verified. +in the current release's tracking issue or merged release PR. Keep any release +tracking issue open until publication and consumer verification succeed. The benchmark producer has read-only repository permissions and restores no dependency caches, including tool binaries. It invokes `setup-tools` with @@ -464,13 +465,13 @@ git branch -d "release/$TAG" git push origin --delete "release/$TAG" ``` -## Configure Trusted Publishing before v0.5.0 +## Configure or verify Trusted Publishing Follow the current [crates.io setup requirements](https://crates.io/docs/trusted-publishing). Trusted Publishing requires an already-published crate and crate ownership; initial publication uses an API token. `la-stack` already exists, so no throwaway -version or initial upload is needed. As its owner, register this exact identity -under the crate's Settings → Trusted Publishing: +version or initial upload is needed. As its owner, verify this exact identity +under the crate's Settings → Trusted Publishing and register it if missing: | Field | Value | | --- | --- | @@ -479,9 +480,9 @@ under the crate's Settings → Trusted Publishing: | Workflow filename | `publish-crates.yml` | | Environment | `crates-io` | -Create GitHub's `crates-io` environment with required maintainer reviewers, +For initial setup, create GitHub's `crates-io` environment with required maintainer reviewers, administrator bypass disabled, and selected deployment tags matching `v*` only -(no branches). The proposed `.github/settings/crates-io-environment.json` names +(no branches). The `.github/settings/crates-io-environment.json` payload names `acgetchell` (user ID 434849) as the required reviewer and permits that maintainer to approve their own deployment; self-review prevention needs another eligible reviewer. `.github/settings/crates-io-tags.json` declares the tag restriction. @@ -495,12 +496,12 @@ excluded from Dependabot automatic approval so these changes require maintainer review. Do not add a long-lived `CARGO_REGISTRY_TOKEN` secret. -Workflow YAML and settings payloads do not apply account configuration. On -October 15, 2026, the GitHub environments API returned no environments and the -authenticated crates.io settings showed no trusted publishers. These payloads -and the publisher registration remain unapplied. Before release, record the actual crates.io -publisher identity and GitHub environment reviewers/tag policy, allowlist and -permissions evidence in #259. Re-read live settings before changing them. +Workflow YAML and settings payloads do not apply account configuration. The initial +Trusted Publishing rollout and verified v0.5.0 publication are recorded in +[#259](https://github.com/acgetchell/la-stack/issues/259). Before each release, +verify the actual crates.io publisher identity and GitHub environment reviewers/tag +policy, allowlist and permissions. Record that evidence in the current release's +tracking issue or release PR. Re-read live settings before changing them. Validate the source with `just lint-config`, `just python-ci`, `just markdown-ci` and `just doc-check`; PR tests use simulated providers and never upload crates. Native CI and successful approved publication remain separate rollout evidence. diff --git a/docs/code_organization.md b/docs/code_organization.md index e1a9d02..d6d16a9 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -192,11 +192,12 @@ gates for both local measurements and release inventories. `tooling/performance.toml` uses the shared measurement schema for source/harness inputs, probes, timeouts, and compatibility fields. `tooling/performance-policy.toml` owns the scientific metric catalog, exact groups, -peer mappings and historical API availability. `scripts/performance_runs.py` -binds those declarations to scientific coverage, release compatibility and +peer mappings and the minimum release for new measurements. `scripts/performance_runs.py` +binds those declarations to scientific coverage, measurement eligibility and named reference-phase assertions. `scripts/archive_performance.py` composes -shared APIs for tracked-only measurements, metadata-free historical raw archives -and la-stack README publication. +shared APIs for tracked-only measurements and la-stack README publication. +Old API cfg and root-test layout adapters and the metadata-free historical +asset comparison command are retired; original archives remain unchanged. `tooling/performance-plot.toml` declares dimensions, units, series and axes. `tooling/performance-report.toml` owns the canonical report path, title, and shared history path. The obsolete CSV conversion declaration and pre-Just Python phase replay are removed. diff --git a/docs/dev/performance-tooling-adoption.md b/docs/dev/performance-tooling-adoption.md index a2f10d2..d8c97ce 100644 --- a/docs/dev/performance-tooling-adoption.md +++ b/docs/dev/performance-tooling-adoption.md @@ -71,17 +71,24 @@ metadata. They cannot provide complete-run samples or newly captured host proven Only two production modules remain: - `archive_performance.py`: la-stack release eligibility, tracked-only snapshots, - historical raw archives without metadata, and README publication composition. -- `performance_runs.py`: captured old-API cfg/environment, scientific selection, + and README publication composition. +- `performance_runs.py`: captured measurement environment, scientific selection, sampling, originating reference phases and source/harness eligibility. -These are consumer policies rather than shared implementations. v0.4.4/v0.4.5 -need the historical cfg and root-test layout adapter. Shared CLI snapshots include -untracked files; la-stack's established contract excludes them. The shared plan -cannot select cfg and API availability from a release pair. Raw historical release -archives lack the metadata required by the shared asset-evidence CLI. Their -small adapter uses shared parsing/rendering and explicitly identifies unrecorded -source, host and gates without constructing an evidence envelope. +New measurements require v0.5.0 or newer on both sides. +The historical API cfg, missing-row exemptions, root-test layout adapter and +metadata-free `performance-github-assets` command are retired. Their Rust +benchmark branches are removed while the independent scientific gates remain. +Original reports, CSV/JSON, plots and archives retain their bytes and hashes. +The measurement floor does not block offline use of retained complete runs +satisfying the modern scientific policy. + +Shared CLI snapshots include untracked files; la-stack's established contract +excludes them. Independent scientific eligibility, saved-sample storage checks +and publication composition also remain local. +[Upstream #115](https://github.com/acgetchell/research-repo-tools/issues/115) +targets declarative replacements in v0.2.1. Adoption and deletion of the final +production Python modules follow publication of that package. Generic subprocesses, discovery parsing, worktrees, measurement, completeness, estimates, comparison arithmetic, rendering, run identity, history, downloads, @@ -143,5 +150,5 @@ Rust tests/doctests, docs, examples, benchmark compilation and native static che No new scientific measurements or regenerated historical artifacts are claimed. Issue closure still requires an adoption commit, native Linux/macOS/Windows CI -for that exact commit, linked run evidence and merge before v0.5.0. Local macOS +for that exact commit, linked run evidence and merge. Local macOS validation does not substitute for native Linux or Windows results. diff --git a/justfile b/justfile index 9407319..b4356c8 100644 --- a/justfile +++ b/justfile @@ -384,10 +384,6 @@ markdown-fix: tools-check performance-doc: python-sync {{ _run }} uv run --locked scripts/archive_performance.py report -# Compare published release archives with their original per-release harnesses. -performance-github-assets current_tag="" baseline_tag="": _ensure-gh python-sync - {{ _run }} uv run --locked scripts/archive_performance.py assets {{ quote(current_tag) }} {{ quote(baseline_tag) }} - # Measure tracked current inputs and the latest release with the captured current harness. performance-local: _ensure-gh python-sync {{ _run }} uv run --locked scripts/archive_performance.py local diff --git a/pyproject.toml b/pyproject.toml index d6234ea..3975c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ line-length = 160 [tool.uv] package = false -required-version = "==0.13.2" +required-version = "==0.13.3" # Semgrep has its own locked Python 3.14 environment under tooling/semgrep. [dependency-groups] diff --git a/scripts/README.md b/scripts/README.md index eebdb01..addc898 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -23,9 +23,9 @@ interpretation, output paths and release procedures. | Owner | Responsibility | | --- | --- | -| `archive_performance.py` | Release eligibility, tracked-only snapshots, historical raw-archive composition, and la-stack README markers/pointers | -| `performance_runs.py` | Historical API cfg/environment, scientific coverage, sampling and original reference-phase assertions | -| `tooling/performance-policy.toml` | Exact group selections, metric names, peers, dimensions and historical API availability | +| `archive_performance.py` | Release eligibility, tracked-only snapshots, and la-stack README markers/pointers | +| `performance_runs.py` | Measurement environment, scientific coverage, sampling and original reference-phase assertions | +| `tooling/performance-policy.toml` | Exact group selections, metric names, peers, dimensions and minimum measurement release | | `tooling/performance.toml` | Shared sources, harness, probes, dependencies, timeouts and compatibility | | `tooling/performance.just` | Native Cargo discovery, independent correctness gates and timing commands | | `tooling/performance-report.toml` | Shared current report and immutable complete-run history | @@ -35,17 +35,20 @@ interpretation, output paths and release procedures. The adapters call public shared APIs for subprocesses, worktrees, Criterion discovery, completeness, estimates, comparison arithmetic, evidence, rendering, -retention, publication, archive extraction and authenticated downloads. They +retention and publication. They contain no generic renderer, CSV parser, legacy writer or measurement engine. -The remaining composition is required by la-stack's contracts: versions before -v0.4.4 are ineligible; v0.4.4/v0.4.5 need the captured old-API cfg and root-test -layout adapter; local measurements exclude untracked files; nalgebra/faer -reference views retain their baseline origin. The shared CLI's default worktree -snapshot includes untracked files and its declarations cannot select an adapter -from a release pair. Historical release archives lack the metadata required by -the shared asset-evidence CLI, so their small adapter renders descriptive -comparisons without inventing source/host/gate provenance. +New measurements require v0.5.0 or newer on both sides. Historical API cfg, +missing-row exemptions, root-test layout adapters and the metadata-free +`performance-github-assets` command are retired. Archived artifacts remain +unchanged; retained complete runs satisfying the modern policy still render +offline regardless of the measurement floor. + +The remaining composition preserves tracked-only local snapshots, independent +scientific eligibility checks and baseline-origin nalgebra/faer reference views. +The shared CLI's default worktree snapshot includes untracked files. +[Upstream #115](https://github.com/acgetchell/research-repo-tools/issues/115) +tracks the declarative interfaces needed to retire these final adapters. `bench-compare` now calls the shared Criterion CLI directly. It compares every available row, including additions and omissions, as an exploratory local report. diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index 633a1b6..ea10aeb 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -14,7 +14,6 @@ from pathlib import Path from typing import Literal -from research_repo_tools.archives import extract_archive from research_repo_tools.common_measurement import measure_prepared_pair from research_repo_tools.complete_runs import render_run, run_identity from research_repo_tools.evidence import Evidence, serialize_evidence @@ -22,7 +21,6 @@ from research_repo_tools.postprocess_changelog import MarkdownFormatError, format_markdown from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_command from research_repo_tools.publication import GitCheck, GlobCheck, MarkerPair, RunSelectionCheck, plan_outputs, publish_publication, replace_section -from research_repo_tools.release_assets import download_release_asset from research_repo_tools.release_discovery import normalize_tag, stable_published_releases from research_repo_tools.release_pairs import PairMode, ReleasePair, resolve_pair from research_repo_tools.run_presentation import ( @@ -41,13 +39,13 @@ from performance_runs import ( REPORT_CONFIG, archive_directory, - compatibility_adapter, find_project_root, measurement_plan, required_report_ids, retained_run, scientific_policy, scratch_paths, + validate_measurement_pair, validate_scientific_run, ) @@ -69,12 +67,13 @@ def select_pair(root: Path, mode: PairMode, current: str | None, baseline: str | pair = ReleasePair(current, baseline) else: pair = resolve_pair(mode, package_tag=package, releases=releases, current=current, baseline=baseline, order="published") - compatibility_adapter(pair, root) + validate_measurement_pair(pair, root) return pair def measure(root: Path, pair: ReleasePair, suite: str, scope: str) -> Evidence: """Capture tracked changes and delegate isolated phases to the published engine.""" + validate_measurement_pair(pair, root) version = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8"))["package"]["version"] if pair.current != normalize_tag(version): msg = "current release label must match the measured checkout package" @@ -90,9 +89,6 @@ def measure(root: Path, pair: ReleasePair, suite: str, scope: str) -> Evidence: temporary_worktree(root, parent / "current", snapshot.revision, allow_git_mutations=True) as current, ): apply_snapshot(current, snapshot) - # Consumer API-layout adapter: the comparison test moved to its own - # workspace member and the historical root copy must not compile. - (baseline / "tests/vs_linalg_inputs.rs").unlink(missing_ok=True) plan = measurement_plan(current, pair, suite, scope) evidence = measure_prepared_pair(baseline, current, plan, pair, working_tree=True) if replace(capture_snapshot(root), untracked=()) != snapshot: @@ -141,31 +137,6 @@ def promote(root: Path, stem: Path) -> None: publish_publication(history) -def compare_assets(root: Path, pair: ReleasePair, output: Path) -> None: - """Compare raw historical archives without inventing missing provenance.""" - from research_repo_tools.criterion import collect_sample, compare_samples, render_comparison # noqa: PLC0415 - - samples = {} - with tempfile.TemporaryDirectory(prefix="la-stack-assets-") as directory: - parent = Path(directory) - for phase, tag in (("baseline", pair.baseline), ("current", pair.current)): - asset = parent / f"{tag}.tar.gz" - download_release_asset(root, "acgetchell/la-stack", tag, f"la-stack-{tag}-criterion-baseline.tar.gz", asset) - extracted = parent / phase - extract_archive(asset, extracted) - samples[phase] = collect_sample(extracted / "criterion", tag) - comparison = compare_samples(samples["baseline"], samples["current"]) - if not comparison.comparisons: - msg = "release archives have no common benchmark rows" - raise ValueError(msg) - text = ( - f"# Published release timings\n\nBaseline: {pair.baseline}; current: {pair.current}.\n\n" - "Each archive used its original release harness. Source, host and gate provenance are not recorded; " - "ratios are descriptive and require independent hardware/workload review.\n\n" + render_comparison(comparison) - ).encode() - publish_publication(plan_outputs(root, {output.relative_to(root).as_posix(): text}, inputs={"Cargo.toml": (root / "Cargo.toml").read_bytes()})) - - def check_release_sample(root: Path, name: str) -> int: """Require canonical report identities and complete, matching saved bytes.""" from research_repo_tools.criterion_baselines import load_saved_sample, validate_saved_sample # noqa: PLC0415 @@ -187,14 +158,13 @@ def main(argv: list[str] | None = None) -> int: """Keep routine performance commands thin and explicit.""" parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) - for name in ("local", "release", "assets"): + for name in ("local", "release"): command = commands.add_parser(name) command.add_argument("current", nargs="?") command.add_argument("baseline", nargs="?") - if name != "assets": - command.add_argument("--suite", choices=("all", "exact", "vs_linalg"), default="all") - command.add_argument("--scope", choices=("release-signal", "all-benches"), default="release-signal") - command.add_argument("--stem", type=Path, default=Path("target/bench-reports/performance")) + command.add_argument("--suite", choices=("all", "exact", "vs_linalg"), default="all") + command.add_argument("--scope", choices=("release-signal", "all-benches"), default="release-signal") + command.add_argument("--stem", type=Path, default=Path("target/bench-reports/performance")) commands.add_parser("report").add_argument("--stem", type=Path, default=Path("target/bench-reports/performance")) commands.add_parser("release-check").add_argument("current", help="Saved Criterion sample name") args = parser.parse_args(argv) @@ -206,19 +176,16 @@ def main(argv: list[str] | None = None) -> int: if args.command == "report": promote(root, root / args.stem) return 0 - modes: dict[str, PairMode] = {"local": "current-vs-latest", "release": "infer-release", "assets": "published-latest"} + modes: dict[str, PairMode] = {"local": "current-vs-latest", "release": "infer-release"} mode: PairMode = "explicit" if args.current or args.baseline else modes[args.command] pair = select_pair(root, mode, args.current or None, args.baseline or None, local=args.command == "local") - if args.command == "assets": - compare_assets(root, pair, root / "target/bench-reports/github-assets-performance.md") - else: - stem = root / args.stem - if args.command == "release" and pair.current == pair.baseline: - parser.error("release requires distinct release labels") - evidence = measure(root, pair, args.suite, args.scope) - save_scratch(root, stem, evidence) - if args.command == "release": - promote(root, stem) + stem = root / args.stem + if args.command == "release" and pair.current == pair.baseline: + parser.error("release requires distinct release labels") + evidence = measure(root, pair, args.suite, args.scope) + save_scratch(root, stem, evidence) + if args.command == "release": + promote(root, stem) return 0 except (OSError, ValueError, TypeError, KeyError, ExecutableNotFoundError, subprocess.SubprocessError, ExceptionGroup) as error: print(f"performance: {format_exception_diagnostics(error)}", file=sys.stderr) diff --git a/scripts/performance_runs.py b/scripts/performance_runs.py index ff6013f..0406dd1 100644 --- a/scripts/performance_runs.py +++ b/scripts/performance_runs.py @@ -24,7 +24,6 @@ REPORT_CONFIG = "tooling/performance-report.toml" MEASUREMENT_CONFIG = "tooling/performance.toml" POLICY_CONFIG = "tooling/performance-policy.toml" -NO_API_COMPATIBILITY = "none" type ReleaseVersion = tuple[int, int, int] @@ -42,12 +41,7 @@ class ScientificPolicy: dimensions: tuple[int, ...] minimum_release: ReleaseVersion - modern_api_release: ReleaseVersion - legacy_api_cfg: str library_benchmarks: tuple[str, ...] - unavailable_prefixes: tuple[str, ...] - unavailable_benchmarks: tuple[str, ...] - unavailable_benchmark_prefixes: tuple[str, ...] exact: Mapping[str, tuple[str, ...]] additional: Mapping[str, tuple[str, ...]] peers: Mapping[str, tuple[str, str]] @@ -125,14 +119,6 @@ def scientific_policy(root: Path | None = None) -> ScientificPolicy: msg = "scientific policy requires schema 1" raise ValueError(msg) minimum = _release_threshold(policy.get("minimum-release"), "minimum-release") - modern = _release_threshold(policy.get("modern-api-release"), "modern-api-release") - if modern < minimum: - msg = "modern API release cannot precede the minimum release" - raise ValueError(msg) - cfg = policy.get("legacy-api-cfg") - if not isinstance(cfg, str) or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", cfg) is None: - msg = "legacy API cfg requires a Rust identifier" - raise ValueError(msg) dimensions = _dimensions(policy.get("dimensions")) library = _names(policy.get("library-benchmarks"), "library-benchmarks") exact = {name: _names(names, f"exact.{name}") for name, names in _table(policy.get("exact"), "exact").items()} @@ -150,12 +136,7 @@ def scientific_policy(root: Path | None = None) -> ScientificPolicy: return ScientificPolicy( dimensions=dimensions, minimum_release=minimum, - modern_api_release=modern, - legacy_api_cfg=cfg, library_benchmarks=library, - unavailable_prefixes=_names(policy.get("unavailable-prefixes"), "unavailable-prefixes"), - unavailable_benchmarks=_names(policy.get("unavailable-benchmarks"), "unavailable-benchmarks"), - unavailable_benchmark_prefixes=_names(policy.get("unavailable-benchmark-prefixes"), "unavailable-benchmark-prefixes"), exact=MappingProxyType(exact), additional=MappingProxyType(additional), peers=MappingProxyType(peers), @@ -163,15 +144,14 @@ def scientific_policy(root: Path | None = None) -> ScientificPolicy: ) -def compatibility_adapter(pair: ReleasePair, root: Path | None = None) -> str: - """Reject excluded releases and select the historical API cfg.""" +def validate_measurement_pair(pair: ReleasePair, root: Path | None = None) -> None: + """Reject unsupported releases before any new measurement side effects.""" policy = scientific_policy(root) versions = {phase: tuple(map(int, normalize_tag(tag)[1:].split("."))) for phase, tag in (("baseline", pair.baseline), ("current", pair.current))} if any(version < policy.minimum_release for version in versions.values()): minimum_release = ".".join(map(str, policy.minimum_release)) msg = f"unsupported comparison release; requires v{minimum_release} or newer" raise ValueError(msg) - return policy.legacy_api_cfg if versions["baseline"] < policy.modern_api_release else NO_API_COMPATIBILITY def required_report_ids(root: Path | None = None) -> set[str]: @@ -220,41 +200,26 @@ def selected_ids(suite: str, scope: str, expected: tuple[str, ...], root: Path | return tuple(sorted(required)) -def phase_environment(root: Path, adapter: str = NO_API_COMPATIBILITY) -> tuple[tuple[str, str], ...]: +def phase_environment(root: Path) -> tuple[tuple[str, str], ...]: """Capture code-generation overrides and isolate each phase's output tree.""" toolchain = tomllib.loads((root / "rust-toolchain.toml").read_text(encoding="utf-8"))["toolchain"]["channel"] env = {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "RUSTC", "RUSTC_WRAPPER", "CARGO_BUILD_TARGET") if key in os.environ} env.update(RUSTUP_TOOLCHAIN=os.environ.get("RUSTUP_TOOLCHAIN", toolchain), CARGO_TARGET_DIR="target", CRITERION_HOME="target/criterion") key, separator = ("CARGO_ENCODED_RUSTFLAGS", "\x1f") if "CARGO_ENCODED_RUSTFLAGS" in env else ("RUSTFLAGS", " ") flags = [env.get(key, ""), "--cap-lints=warn"] - if adapter != NO_API_COMPATIBILITY: - flags.append(f"--cfg={adapter}") env[key] = separator.join(flag for flag in flags if flag) return tuple(sorted(env.items())) -def _historical_case_available(full_id: str, policy: ScientificPolicy) -> bool: - """Mirror the groups and norm cases disabled by the historical harness cfg.""" - bench = full_id.partition("/")[2] - return ( - not full_id.startswith(policy.unavailable_prefixes) - and bench not in policy.unavailable_benchmarks - and not bench.startswith(policy.unavailable_benchmark_prefixes) - ) - - def measurement_plan(root: Path, pair: ReleasePair, suite: str, scope: str) -> CommonHarnessPlan: - """Build the shared plan from native inventories and consumer API policy.""" - adapter = compatibility_adapter(pair, root) - policy = scientific_policy(root) - baseline_env = phase_environment(root, adapter) - current_env = phase_environment(root) + """Build the shared plan from complete native inventories.""" + validate_measurement_pair(pair, root) + environment = phase_environment(root) config = configuration(root) inventories = {} for phase in ("baseline", "current"): - # Discover against the current API, then apply the exact groups excluded - # by the historical cfg. Old method names cannot compile on current code. - # The shared collector later checks this entire inventory on each source. + # Both revisions use the captured current harness. The shared collector + # later checks this entire inventory on each measured source. declarations = load_discovery(root, "tooling/criterion-inventory.toml") discovery = DiscoveryPlan( tuple( @@ -263,14 +228,11 @@ def measurement_plan(root: Path, pair: ReleasePair, suite: str, scope: str) -> C if suite in {"all", item.name} ) ) - ids = discover_inventory(root, discovery, env=os.environ | dict(current_env)).expected - old_api = phase == "baseline" and adapter != NO_API_COMPATIBILITY - inventories[phase] = tuple(sorted(name for name in ids if not old_api or _historical_case_available(name, policy))) - return declared_plan(pair, suite, scope, inventories, baseline_env=baseline_env, current_env=current_env, config=config, root=root) + inventories[phase] = discover_inventory(root, discovery, env=os.environ | dict(environment)).expected + return declared_plan(suite, scope, inventories, baseline_env=environment, current_env=environment, config=config, root=root) def declared_plan( # noqa: PLR0913 - pair: ReleasePair, suite: str, scope: str, inventories: dict[str, tuple[str, ...]], @@ -283,15 +245,12 @@ def declared_plan( # noqa: PLR0913 """Bind exact case inventories, reference phases, commands and completeness.""" config = config or configuration(root) policy = scientific_policy(root) - adapter = compatibility_adapter(pair, root) selected = selected_ids(suite, scope, inventories["current"], root) - unavailable = adapter != NO_API_COMPATIBILITY - baseline_rows = tuple(name for name in selected if not unavailable or _historical_case_available(name, policy)) - missing = set(baseline_rows) - set(inventories["baseline"]) + missing = set(selected) - set(inventories["baseline"]) if missing: raise ValueError(f"baseline inventory omits selected rows: {sorted(missing)}") series = [ - RunSeries("la-stack baseline", "baseline", tuple((name, name) for name in baseline_rows)), + RunSeries("la-stack baseline", "baseline", tuple((name, name) for name in selected)), RunSeries("la-stack current", "current", tuple((name, name) for name in selected)), ] peers = policy.peers @@ -317,7 +276,8 @@ def declared_plan( # noqa: PLR0913 *config.context, ("suite", suite), ("scope", scope), - ("baseline-api-compatibility", adapter), + # Preserve the provenance field used by retained modern complete runs. + ("baseline-api-compatibility", "none"), ), ) return CommonHarnessPlan(config, phases["baseline"], phases["current"], tuple(series)) @@ -329,7 +289,6 @@ def validate_scientific_run(evidence: Evidence, *, release: bool = False, root: sources = dict(evidence.sources) context = dict(sources["current"].context) pair = ReleasePair(context["release"], dict(sources["baseline"].context)["release"]) - adapter = compatibility_adapter(pair, root) if release and pair.current == pair.baseline: msg = "cannot promote a same-version local comparison as release documentation" raise ValueError(msg) @@ -341,7 +300,6 @@ def validate_scientific_run(evidence: Evidence, *, release: bool = False, root: msg = "la-stack requires 100 samples, mean and median, and 95% intervals in ns" raise ValueError(msg) expected = declared_plan( - pair, context["suite"], context["scope"], {phase: sample.policy.expected for phase, sample in samples.items()}, @@ -349,8 +307,8 @@ def validate_scientific_run(evidence: Evidence, *, release: bool = False, root: current_env=(), root=root, ) - if run.series != expected.series or context["baseline-api-compatibility"] != adapter: - msg = "retained series or compatibility policy differs from la-stack declarations" + if run.series != expected.series: + msg = "retained series differs from la-stack declarations" raise ValueError(msg) if set(run.compatible) != set(expected.measurement.compatible): msg = "retained run omits required host, toolchain, or dependency compatibility" diff --git a/scripts/tests/test_performance_workflow.py b/scripts/tests/test_performance_workflow.py index e12b385..8bd6ac1 100644 --- a/scripts/tests/test_performance_workflow.py +++ b/scripts/tests/test_performance_workflow.py @@ -1,14 +1,12 @@ """Installed-package acceptance of la-stack's complete performance workflow.""" import hashlib -import io import json import os import shlex import shutil import subprocess import sys -import tarfile import tomllib from dataclasses import replace from pathlib import Path @@ -33,7 +31,7 @@ from research_repo_tools.publication import PublicationPlan ROOT = Path(__file__).resolve().parents[2] -PAIR = ReleasePair("v0.4.6", "v0.4.5") +PAIR = ReleasePair("v0.5.1", "v0.5.0") DRIVER = """import json, sys from pathlib import Path action, phase = sys.argv[1:] @@ -57,7 +55,7 @@ def prepared(tmp_path: Path) -> tuple[Path, Path, CommonHarnessPlan]: """Use the real consumer inventory and policies with a bounded timing fixture.""" ids = tuple(sorted(required_report_ids())) inventories = { - "baseline": tuple(name for name in ids if not name.startswith(policy.scientific_policy().unavailable_prefixes)), + "baseline": ids, "current": tuple(name for name in ids if not name.startswith("d") or "/la_stack_" in name), } for phase in ("baseline", "current"): @@ -109,7 +107,7 @@ def prepared(tmp_path: Path) -> tuple[Path, Path, CommonHarnessPlan]: cwd=root, ) baseline, current = tmp_path / "baseline", tmp_path / "current" - plan = policy.declared_plan(PAIR, "all", "release-signal", inventories, baseline_env=(), current_env=(), config=policy.configuration(current)) + plan = policy.declared_plan("all", "release-signal", inventories, baseline_env=(), current_env=(), config=policy.configuration(current)) return baseline, current, plan @@ -157,7 +155,9 @@ def test_measurement_worktrees_accept_an_aliased_system_temporary_directory(tmp_ """Canonicalize owned temporary paths before the native worktree safety check.""" root = tmp_path.resolve() / "repository" root.mkdir() - (root / "Cargo.toml").write_text('[package]\nname = "fixture"\nversion = "0.4.6"\n', encoding="utf-8") + (root / "Cargo.toml").write_text('[package]\nname = "fixture"\nversion = "0.5.1"\n', encoding="utf-8") + (root / "tooling").mkdir() + shutil.copyfile(ROOT / policy.POLICY_CONFIG, root / policy.POLICY_CONFIG) run_command("git", ["--no-pager", "init", "--quiet"], cwd=root) run_command("git", ["--no-pager", "add", "Cargo.toml"], cwd=root) run_command( @@ -204,13 +204,15 @@ def benchmark_ready(baseline: Path, current: Path, _plan: object, _pair: Release assert list(physical.iterdir()) == [] -def test_consumer_plan_rejects_an_inventory_missing_a_required_report_row() -> None: +@pytest.mark.parametrize("phase", ["baseline", "current"]) +def test_consumer_plan_rejects_an_inventory_missing_a_required_report_row(phase: str) -> None: missing = "d64/la_stack_lu_solve" ids = tuple(sorted(required_report_ids())) assert missing in ids - inventories = {"baseline": ids, "current": tuple(name for name in ids if name != missing)} - with pytest.raises(ValueError, match=rf"inventory omits selected report rows: \['{missing}'\]"): - policy.declared_plan(PAIR, "all", "release-signal", inventories, baseline_env=(), current_env=(), root=ROOT) + inventories = {"baseline": ids, "current": ids} + inventories[phase] = tuple(name for name in ids if name != missing) + with pytest.raises(ValueError, match=rf"inventory omits selected (?:report )?rows: \['{missing}'\]"): + policy.declared_plan("all", "release-signal", inventories, baseline_env=(), current_env=(), root=ROOT) @pytest.mark.parametrize("report_path", ["docs/performance.md", "docs/reports/current.md"]) @@ -301,7 +303,7 @@ def prepare_readme_recipe(root: Path, statistic: str) -> None: def assert_readme_presentation(readme: str, statistic: str) -> None: """Check the consumer's compact style, comparison direction and figure.""" table = readme.split(f"", 1)[1].split(f"", 1)[0] - assert "| Dimension | la-stack (v0.4.6) | nalgebra (" in table + assert f"| Dimension | la-stack ({PAIR.current}) | nalgebra (" in table assert "| faer (" in table assert "| Speedup vs nalgebra | Speedup vs faer |" in table assert "(0.95 confidence)" not in table @@ -314,7 +316,7 @@ def assert_readme_presentation(readme: str, statistic: str) -> None: assert "ratios are point estimates, with no ratio intervals or significance claims" in table for name in ("nalgebra", "faer"): assert f"Speedup vs {name}: ratio; numerator = {name} (" in table - assert table.count("denominator = la-stack (v0.4.6)") == 2 + assert table.count(f"denominator = la-stack ({PAIR.current})") == 2 assert "time reduction" not in table assert f"![LU solve (factor + solve): {statistic} time vs dimension][lu-solve-benchmark]" in readme assert ( @@ -514,7 +516,7 @@ def test_existing_tag_cannot_receive_different_measured_assets(complete_run: tup ], cwd=root, ) - run_command("git", ["--no-pager", "-c", "tag.gpgsign=false", "tag", "v0.4.6"], cwd=root) + run_command("git", ["--no-pager", "-c", "tag.gpgsign=false", "tag", PAIR.current], cwd=root) originals = {path: path.read_bytes() for path in (*assets.iterdir(), root / "README.md")} with pytest.raises(ValueError, match="exact publication artifact"): workflow.publish_plot(root, "lu_solve", "median", log_y=True, update_readme=True) @@ -598,7 +600,28 @@ def test_legacy_bytes_hashes_are_unchanged() -> None: assert {name: hashlib.sha256((directory / name).read_bytes()).hexdigest() for name in hashes} == hashes -def test_native_command_policy_and_compatibility_adapter() -> None: +def test_retained_modern_evidence_below_measurement_floor_remains_readable() -> None: + """The execution floor must not disable replay of complete archived evidence.""" + identity = "634bc9527c2caf0a431a9bdfac082c190e1094f0b29eb6e3f28c4b96961cdec5" + directory = ROOT / "docs/performance-runs/runs" / identity + evidence = policy.load_evidence(directory / "run.json", directory / "evidence.json") + run = policy.validate_scientific_run(evidence, release=True, root=ROOT) + releases = {phase: dict(source.context)["release"] for phase, source in evidence.sources} + assert releases == {"baseline": "v0.4.6", "current": "v0.5.0"} + with pytest.raises(ValueError, match=r"requires v0\.5\.0 or newer"): + policy.validate_measurement_pair(ReleasePair(releases["current"], releases["baseline"]), ROOT) + assert {series.phase for series in run.series if series.name in {"nalgebra", "faer"}} == {"baseline"} + report = render_run( + evidence, + report_path="docs/performance.md", + payload_path=f"docs/performance-runs/runs/{identity}/run.json", + manifest_path=f"docs/performance-runs/runs/{identity}/evidence.json", + ) + assert identity.encode() in report + assert b"v0.4.6" in report + + +def test_native_command_policy_and_phase_environment() -> None: justfile = Path("tooling/performance.just") baseline = dry_run(ROOT, "baseline-all", justfile=justfile).stderr current = dry_run(ROOT, "current-all", justfile=justfile).stderr @@ -610,13 +633,12 @@ def test_native_command_policy_and_compatibility_adapter() -> None: assert dry_run(ROOT, "baseline-all", ["list"], justfile=justfile).stderr.count("--list") == 2 gate = dry_run(ROOT, "gate", justfile=justfile).stderr assert "--test vs_linalg_inputs --test exact_bench_config" in gate - environment = dict(policy.phase_environment(ROOT, "la_stack_pre_rational_input_api")) - assert "--cfg=la_stack_pre_rational_input_api" in environment.get("CARGO_ENCODED_RUSTFLAGS", environment.get("RUSTFLAGS", "")) + environment = dict(policy.phase_environment(ROOT)) assert environment["CARGO_TARGET_DIR"] == "target" assert environment["CRITERION_HOME"] == "target/criterion" -@pytest.mark.parametrize("baseline_tag", ["v0.4.4", "v0.4.5", "v0.4.6"]) +@pytest.mark.parametrize("baseline_tag", ["v0.5.0", "v0.5.1"]) @pytest.mark.parametrize("scope", ["release-signal", "all-benches"]) def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, baseline_tag: str, scope: str) -> None: """Exercise the configured Justfile, stubbing only the expensive Cargo boundary.""" @@ -627,8 +649,7 @@ def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: binary = tmp_path / "bin" binary.mkdir() cargo = binary / "cargo" - # These are the registrations guarded by the historical API cfg in - # benches/vs_linalg.rs, including the two reference scenario kernels. + # Diagnostic norm registrations include the two reference scenario kernels. norm_rows = { f"d{dimension}/{bench}" for dimension in (2, 3, 4, 5) @@ -666,17 +687,16 @@ def test_native_inventory_and_phase_commands_use_the_measured_checkout(tmp_path: monkeypatch.setenv("CARGO_RECORD", str(record)) monkeypatch.setenv("CRITERION_HOME", "target/criterion") monkeypatch.chdir(tmp_path) - pair = ReleasePair("v0.4.6", baseline_tag) + pair = ReleasePair(PAIR.current, baseline_tag) plan = policy.measurement_plan(root, pair, "all", scope) assert isinstance(plan.current.policy, CompletePolicy) assert isinstance(plan.baseline.policy, CompletePolicy) assert set(plan.current.policy.expected) == set(current_ids) | set(exact) - unavailable = norm_rows | {name for name in exact if name.startswith(("rational_input_", "canonical_conversion_", "det4_diagnostic_"))} - assert set(plan.baseline.policy.expected) == (ids if baseline_tag == "v0.4.6" else ids - unavailable) + assert set(plan.baseline.policy.expected) == ids assert ordinary_norm_peers <= set(plan.baseline.policy.expected) assert "d2/la_stack_norm2_sq" in plan.baseline.policy.expected series = {item.name: {name for name, _ in item.rows} for item in plan.series} - assert series["la-stack baseline"] == (series["la-stack current"] if baseline_tag == "v0.4.6" else series["la-stack current"] - unavailable) + assert series["la-stack baseline"] == series["la-stack current"] if scope == "all-benches": assert "d2/la_stack_norm2" in series["la-stack current"] assert "d2/la_stack_norm2_scenario_sparse" in series["la-stack current"] @@ -735,11 +755,30 @@ def test_retired_python_phase_format_is_rejected(tmp_path: Path) -> None: def test_local_explicit_labels_and_release_eligibility(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) - assert workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.6", local=True) == ReleasePair("v0.4.6", "v0.4.6") + assert workflow.select_pair(ROOT, "explicit", "v0.5.0", "v0.5.0", local=True) == ReleasePair("v0.5.0", "v0.5.0") with pytest.raises(ValueError, match="must differ"): - workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.6") - with pytest.raises(ValueError, match=r"v0.4.4|unsupported"): - workflow.select_pair(ROOT, "explicit", "v0.4.6", "v0.4.3", local=True) + workflow.select_pair(ROOT, "explicit", "v0.5.0", "v0.5.0") + assert workflow.select_pair(ROOT, "explicit", PAIR.current, PAIR.baseline) == PAIR + + +@pytest.mark.parametrize("old_tag", ["v0.4.3", "v0.4.4", "v0.4.5", "v0.4.6", "v0.4.99"]) +@pytest.mark.parametrize("phase", ["baseline", "current"]) +def test_old_releases_are_rejected_before_measurement_side_effects(old_tag: str, phase: str, monkeypatch: pytest.MonkeyPatch) -> None: + pair = ReleasePair(PAIR.current, old_tag) if phase == "baseline" else ReleasePair(old_tag, PAIR.baseline) + + def forbidden(*_args: object, **_kwargs: object) -> None: + pytest.fail("unsupported releases must fail before snapshot, fetch, worktree creation or discovery") + + monkeypatch.setattr(workflow, "capture_snapshot", forbidden) + monkeypatch.setattr(workflow, "run_command", forbidden) + monkeypatch.setattr(policy, "discover_inventory", forbidden) + for action in ( + lambda: workflow.select_pair(ROOT, "explicit", pair.current, pair.baseline, local=True), + lambda: workflow.measure(ROOT, pair, "all", "release-signal"), + lambda: policy.measurement_plan(ROOT, pair, "all", "release-signal"), + ): + with pytest.raises(ValueError, match=r"requires v0\.5\.0 or newer"): + action() @pytest.mark.parametrize( @@ -749,9 +788,6 @@ def test_local_explicit_labels_and_release_eligibility(tmp_path: Path, monkeypat ("minimum-release", "[0, 4]"), ("minimum-release", "[0, 4, true]"), ("minimum-release", "[0, 4, -1]"), - ("modern-api-release", "[]"), - ("modern-api-release", "[0, 4, 3]"), - ("legacy-api-cfg", '"--arbitrary-flag"'), ], ) def test_malformed_release_policy_is_rejected(tmp_path: Path, field: str, value: str) -> None: @@ -762,8 +798,8 @@ def test_malformed_release_policy_is_rejected(tmp_path: Path, field: str, value: text = re.sub(rf"(?m)^{re.escape(field)} = .*$", f"{field} = {value}", text) (tmp_path / policy.POLICY_CONFIG).write_text(text, encoding="utf-8") - with pytest.raises(ValueError, match=r"release|cfg"): - policy.compatibility_adapter(PAIR, tmp_path) + with pytest.raises(ValueError, match="minimum-release"): + policy.validate_measurement_pair(PAIR, tmp_path) @pytest.mark.parametrize( @@ -776,9 +812,6 @@ def test_malformed_release_policy_is_rejected(tmp_path: Path, field: str, value: ("dimensions", "[2, 2]", "dimensions"), ("library-benchmarks", '"la_stack_lu"', "library-benchmarks"), ("library-benchmarks", '["la_stack_lu", "la_stack_lu"]', "library-benchmarks"), - ("unavailable-prefixes", "[false]", "unavailable-prefixes"), - ("unavailable-benchmarks", '["../escape"]', "unavailable-benchmarks"), - ("unavailable-benchmark-prefixes", "[1]", "unavailable-benchmark-prefixes"), ("exact_d2", '"det"', "exact.exact_d2"), ('"8"', "[false]", "additional.8"), ("la_stack_lu", '["nalgebra_lu"]', "peers.la_stack_lu"), @@ -799,7 +832,7 @@ def test_malformed_scientific_catalog_is_rejected(tmp_path: Path, field: str, va assert count == 1 (tmp_path / policy.POLICY_CONFIG).write_text(text, encoding="utf-8") with pytest.raises(ValueError, match=re.escape(diagnostic)): - policy.compatibility_adapter(PAIR, tmp_path) + policy.scientific_policy(tmp_path) @pytest.mark.parametrize( @@ -833,12 +866,11 @@ def test_parsed_scientific_catalog_preserves_immutable_peer_and_metric_rows() -> cast("dict[str, tuple[str, str]]", catalog.peers)["la_stack_lu"] = ("wrong", "peer") -@pytest.mark.parametrize("flag", [("--suite", "exact"), ("--scope", "all-benches"), ("--stem", "ignored")]) -def test_assets_rejects_retired_options(flag: tuple[str, str], capsys: pytest.CaptureFixture[str]) -> None: +def test_historical_asset_command_is_retired(capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit) as error: - workflow.main(["assets", *flag]) + workflow.main(["assets"]) assert error.value.code == 2 - assert "unrecognized arguments" in capsys.readouterr().err + assert "invalid choice: 'assets'" in capsys.readouterr().err @pytest.mark.parametrize("command", ["plot", "local"]) @@ -868,14 +900,11 @@ def test_reference_phase_and_same_version_policy(tmp_path: Path) -> None: policy.validate_scientific_run(replace(evidence, payload=serialize_run(replace(run, series=series)))) # Use a current/current release pair with both sources under the same API. baseline, current, plan = prepared(tmp_path / "same") - pair = ReleasePair("v0.4.6", "v0.4.6") + pair = ReleasePair(PAIR.current, PAIR.current) inventories = {phase: tuple(case.full_id for case in sample.cases) for phase, sample in run.phases} - # The older-API fixture deliberately lacks rows; removing them from the - # current selection allows a narrow all-benches same-version comparison. - inventories["current"] = tuple(name for name in inventories["current"] if name in inventories["baseline"]) driver = f"INVENTORIES = {inventories!r}\n" + DRIVER (current / "tooling/fixture.py").write_text(driver, encoding="utf-8") - plan = policy.declared_plan(pair, "all", "all-benches", inventories, baseline_env=(), current_env=(), config=policy.configuration(current)) + plan = policy.declared_plan("all", "all-benches", inventories, baseline_env=(), current_env=(), config=policy.configuration(current)) same = measure_prepared_pair(baseline, current, plan, pair, working_tree=True) policy.validate_scientific_run(same) with pytest.raises(ValueError, match="same-version"): @@ -888,22 +917,6 @@ def test_reference_phase_and_same_version_policy(tmp_path: Path) -> None: policy.validate_scientific_run(identical) -def test_unsafe_release_asset_cannot_replace_report(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - output = tmp_path / "report.md" - output.write_bytes(b"previous report\n") - - def unsafe_asset(_root: Path, _repo: str, _tag: str, _name: str, destination: Path) -> None: - with tarfile.open(destination, "w:gz") as archive: - member = tarfile.TarInfo("../../report.md") - member.size = 4 - archive.addfile(member, io.BytesIO(b"evil")) - - monkeypatch.setattr(workflow, "download_release_asset", unsafe_asset) - with pytest.raises(ValueError, match=r"unsafe|traversal|relative|member"): - workflow.compare_assets(tmp_path, PAIR, output) - assert output.read_bytes() == b"previous report\n" - - def test_shared_saved_comparison_recipe_reports_coverage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Exercise the actual recipe and installed CLI, with no consumer package.""" for name in ("justfile", "pyproject.toml", "uv.lock", ".python-version", "rust-toolchain.toml"): diff --git a/tests/exact_bench_config.rs b/tests/exact_bench_config.rs index fbeaa13..53ae8fa 100644 --- a/tests/exact_bench_config.rs +++ b/tests/exact_bench_config.rs @@ -8,18 +8,14 @@ mod bench_utils; #[path = "../benches/common/exact.rs"] pub mod exact_bench; -#[cfg(not(la_stack_pre_rational_input_api))] #[path = "../benches/common/exact_diagnostics.rs"] pub mod exact_diagnostics; -#[cfg(not(la_stack_pre_rational_input_api))] #[path = "../benches/common/rational.rs"] pub mod rational_bench; -#[cfg(not(la_stack_pre_rational_input_api))] #[path = "../benches/common/rational_solve.rs"] pub mod rational_solve; -#[cfg(not(la_stack_pre_rational_input_api))] #[test] fn structured_rational_solve_fixtures_are_correct() { fn check() { @@ -51,7 +47,6 @@ use exact_bench::{ large_entries_3x3_input, make_matrix_rows, make_random_input_corpus, make_vector_array, near_singular_3x3_input, validate_exact_fixture, validate_f64_determinant_benchmarks, }; -#[cfg(not(la_stack_pre_rational_input_api))] use exact_diagnostics::{ConversionKind, Det4Kind, canonical_conversion_input, exact_det4_input}; fn baseline_input() -> ExactInput { @@ -65,7 +60,6 @@ fn baseline_input() -> ExactInput { } fn validate_baseline_and_random_corpus() { - #[cfg(not(la_stack_pre_rational_input_api))] for kind in ConversionKind::ALL { let _ = canonical_conversion_input::(kind); } @@ -76,7 +70,6 @@ fn validate_baseline_and_random_corpus() { } } -#[cfg(not(la_stack_pre_rational_input_api))] #[test] fn determinant_diagnostic_fixtures_are_correct() { for kind in Det4Kind::ALL { @@ -84,7 +77,6 @@ fn determinant_diagnostic_fixtures_are_correct() { } } -#[cfg(not(la_stack_pre_rational_input_api))] #[test] fn determinant_exponent_spreads_preserve_exact_values() { use exact_diagnostics::{EXPONENT_SPREADS, ExponentAxis, exponent_spread_input}; @@ -103,7 +95,6 @@ fn determinant_exponent_spreads_preserve_exact_values() { } /// Keep the filter-resolved control and typed non-finite fallback distinct. -#[cfg(not(la_stack_pre_rational_input_api))] #[test] fn determinant_extreme_diagnostic_filter_paths_are_stable() { use la_stack::{ArithmeticOperation, LaError}; diff --git a/tooling/performance-policy.toml b/tooling/performance-policy.toml index 1572363..bd34dd0 100644 --- a/tooling/performance-policy.toml +++ b/tooling/performance-policy.toml @@ -1,13 +1,8 @@ # la-stack scientific selections; generic execution and rendering are shared. schema = 1 dimensions = [ 2, 3, 4, 5, 8, 16, 32, 64 ] -minimum-release = [ 0, 4, 4 ] -modern-api-release = [ 0, 4, 6 ] -legacy-api-cfg = "la_stack_pre_rational_input_api" +minimum-release = [ 0, 5, 0 ] library-benchmarks = [ "la_stack_det", "la_stack_det_from_ldlt", "la_stack_det_from_lu", "la_stack_det_via_lu", "la_stack_dot", "la_stack_inf_norm", "la_stack_ldlt", "la_stack_ldlt_solve", "la_stack_lu", "la_stack_lu_solve", "la_stack_norm2_sq", "la_stack_solve_from_ldlt", "la_stack_solve_from_lu" ] -unavailable-prefixes = [ "rational_input_", "canonical_conversion_", "det4_diagnostic_" ] -unavailable-benchmarks = [ "la_stack_norm2" ] -unavailable-benchmark-prefixes = [ "la_stack_norm2_scenario_", "iterative_f64_hypot_norm2_scenario_", "delaunay_scaled_norm_norm2_scenario_" ] [exact] exact_d2 = [ "det", "det_direct", "det_direct_with_errbound", "det_errbound", "det_exact", "det_exact_f64_result", "det_exact_rounded_f64", "det_sign_exact", "solve_exact", "solve_exact_f64_result", "solve_exact_rounded_f64" ]