From b9fdb60dc5ba1133a44b3003011768e107209568 Mon Sep 17 00:00:00 2001 From: voidwest Date: Thu, 22 Oct 2026 22:28:27 +0300 Subject: [PATCH 1/2] feat(sweep): sweep the features of a dictionary Add `features = { dictionary, ids }` to the `[sweep]` table. Each point gives the swept interventions `features = { dictionary, ids = [F] }`, so each point acts on one feature. `ids` is a list or a range. There is no "all". The swept interventions must be at the dictionary's site and layer, and must use an operation that acts on features. The sweep cannot also move layers, heads or positions. It can cross with `alphas`. A sweep sets at most 65,536 features, and the range is counted before it is collected. `run_sweep` checks every id against the dictionary's width before the first point runs. `sweep.json` records `features` and each point's `feature`. `sweep.csv` and `sweep-effect.csv` get a `feature` column for feature sweeps only. Sweeps without `features` write the same files and `sweep_hash` as before. `verify` compares the recorded features with the definition and with each point. `validate` shows the feature axis. Audit record: docs/audits/2026-10-22-feature-sweeps.md. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 + docs/audits/2026-10-22-feature-sweeps.md | 111 +++++++ docs/audits/README.md | 2 +- docs/experiment-schema-v1.md | 8 +- docs/experiments.md | 22 ++ src/cli_experiment_dictionary.rs | 4 +- src/cli_experiment_sweep.rs | 137 ++++++++ src/v05/sweep.rs | 380 +++++++++++++++++++++-- 8 files changed, 649 insertions(+), 25 deletions(-) create mode 100644 docs/audits/2026-10-22-feature-sweeps.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dcc6d4..4bb0511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,16 @@ Work after the 1.0.0 candidate. It is not part of the staged 1.0.0 release. ### Added +- Sweeps: `features = { dictionary, ids }` in `[sweep]` sweeps the features + of a feature dictionary, one per point: each point gives the swept + interventions `features = { dictionary, ids = [F] }`. `ids` is a list or + a range. The swept interventions stay at the dictionary's site and layer, + so a feature sweep crosses only with `alphas`. `sweep.json` records + `features` and each point's `feature`, and `sweep.csv` and + `sweep-effect.csv` gain a `feature` column for feature sweeps only. + Sweeps without `features` write the same files and `sweep_hash` as + before. `examples/experiments/capital-feature-sweep.toml` sweeps eight + country features across eight capital prompts. - Feature dictionaries: `scripts/convert_linear_sae.py` converts a ReLU SAE saved as a PyTorch state dict (`torch.nn.Linear` layout) to a SAELens folder. It reads the `.pt` without running its pickle and refuses every diff --git a/docs/audits/2026-10-22-feature-sweeps.md b/docs/audits/2026-10-22-feature-sweeps.md new file mode 100644 index 0000000..91f25a6 --- /dev/null +++ b/docs/audits/2026-10-22-feature-sweeps.md @@ -0,0 +1,111 @@ +# Sweeps audit (feature sweeps): 2026-10-22 + +- **Revision/branch:** `feat/feature-sweep` on `main` at `a5f42ae`. +- **Audit date:** 2026-10-22 +- **Scope and trigger:** layer sweeps (`src/v05/sweep.rs`, + `src/cli_experiment_sweep.rs`) and feature interventions + (`src/v05/dictionary.rs`, `src/cli_experiment_dictionary.rs`). Trigger: a + public schema change. The `[sweep]` table gains `features`, and + `ember.sweep.v1` gains optional `features` and per-point `feature` fields. +- **Primary:** research maintainer (voidwest). +- **Backup:** not assigned (open since the 2026-10-11 record). +- **Second reviewer:** an automated review agent read the diff without a + build. No human second reviewer. +- **Last audit / next audit:** last: `2026-10-21-head-sweeps.md` (sweeps) + and `2026-10-22-feature-dictionaries.md` (dictionaries). Next: before the + next release tag, or at the next `ember.sweep.v1` change. + +## 1. Map and boundaries + +- User-facing commands/APIs: `ember experiment validate|run|verify|compare|inspect` + with `features = { dictionary, ids }` in `[sweep]`. +- Source entry points: `SweepDefinition::parse` and `parse_features`; + `point_count` and `points`; the feature check in `run_sweep`; + `verify_and_load_sweep`. +- Adjacent subsystems and handoffs: feature interventions (ADR-0003); the + dictionary cache in `load_dictionaries`; the shared pass and prefix reuse. +- Persistent artifacts or schemas: `sweep.json` (`features`, per-point + `feature`), `sweep.csv` and `sweep-effect.csv` (`feature` column only for + feature sweeps). Point bundles are ordinary `ember.bundle.v1` bundles with + `features = { dictionary, ids = [F] }` on the swept interventions. +- Environment switches and feature gates: none new. +- External references or trusted implementations: none. +- Supported targets and deliberate non-goals: no `ids = "all"`. A feature + sweep does not also move layers, heads or positions, because a dictionary + is trained at one site and layer. It can cross with `alphas`. + +## 2. Contracts and invariants + +- Ownership/lifetime and resource limits: a sweep sets at most 65,536 + features (`MAX_SWEPT_FEATURES`, the width of the largest dictionary that + the weights cap admits). A range is counted before it is collected. The + run checks every id against the dictionary's `d_sae` before the first + point. The points reuse the cached weights. +- Shape, encoding, safety, or error behavior: the swept interventions must + be at the dictionary's site and layer, must not declare `features`, and + must use `zero`, `scale`, `replace`, `add-delta` or `interpolate`. An + unknown dictionary, an empty list or a malformed range fails at parse + time, with the `sweep.features` path. +- Numerical, determinism, trace, or hook boundaries: no change to hooks or + to the feature-space arithmetic. Each point bundle is the bundle that its + derived spec gives alone. +- Compatibility/provenance requirements: sweeps without `features` write + the same `sweep.json`, CSV files and `sweep_hash`; the frozen fixtures + (`tests/sweep_compatibility.rs`) still verify. `verify` compares the + recorded `features` with the definition and each point's `feature` with + its coordinates. +- Reference/oracle path versus optimized path: not applicable. + +## 3. Checks and evidence + +Host: Apple M1 Pro, macOS, Rust 1.98.1 (GUI) and 1.92.0 (headless MSRV), +`CARGO_BUILD_JOBS=2`. + +| Check | Command / fixture | Result | Evidence or artifact | +| --- | --- | --- | --- | +| Static/build | `scripts/ci_local.sh quick` (part of `full`) | pass | fmt, clippy (all configurations), MSRV, docs | +| Focused unit/integration | `feature_sweeps_set_one_feature_per_point`, `feature_sweeps_refuse_what_a_dictionary_cannot_do`, `feature_sweeps_run_one_feature_per_point_and_verify` | pass | grid order and point ids; every refusal; a forged feature or id list fails `verify`; an id past `d_sae` fails before any bundle | +| Full suites | `scripts/ci_local.sh full` | pass: 1,760 Rust tests and 146 Python tests (1 skipped); `cargo audit` not run (not installed locally; CI runs it) | all suites | +| Compatibility | `tests/sweep_compatibility.rs` | pass | frozen sweeps verify with their recorded hashes | +| Smoke | `capital-feature-sweep.toml` on Llama-3.2-1B-Instruct Q8_0 | pass | 9 bundles verify; 2.5 s of compute; 38 MB; a second run gives the same `sweep-effect.csv` | +| Golden logits / external reference | not run | n/a | no inference change | +| Performance (controlled A/B) | not run | n/a | no hot-path change | +| Backup reproduction | not run | n/a | no backup assigned | + +Model: `Llama-3.2-1B-Instruct-Q8_0.gguf`, SHA-256 +`432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3`. +Dictionary: the qresearch layer-9 SAE converted to SAELens, weights SHA-256 +`8ade5dc5488d13676a26cad16b4d9bf88c947b9f6df44441996fe87ffff6977f`. The +dictionary is local only and is not in the repository. + +## 4. Findings + +| Severity | Finding / evidence | Owner or issue | Due / disposition | +| --- | --- | --- | --- | +| required | A range such as `{ start = 0, end = 9223372036854775807, step = 1 }` was collected before any limit, so `validate` could exhaust memory. | research maintainer | fixed before merge: the range is counted first, and a sweep sets at most 65,536 features | +| required | A swept `steer`, `ablate-projection` or `restore-original` intervention passed the sweep parse and failed only when the points were derived, after the model loaded. | research maintainer | fixed before merge: refused at parse time with the `sweep.features` path | +| required | An id past the dictionary's width failed only when its point ran, after the earlier points wrote bundles. | research maintainer | fixed before merge: `run_sweep` checks every id against `d_sae` before the first point | +| observation | `validate` did not show the feature axis. | research maintainer | fixed before merge: the JSON output has `features`, and the text output names the count and the dictionary | +| observation | The first docs said that each feature moves only its own country's prompt. Six off-diagonal entries are not 0. | research maintainer | fixed before merge: the example header explains each of the six from a full feature capture of the rows | +| follow-up | No named backup for the sweeps surface. | project owner | assign at the next audit | + +## 5. Newcomer handoff + +A feature sweep keeps the swept interventions at the dictionary's site and +layer and gives each point one feature: `features = { dictionary, ids = +[F] }`. Start with +`cargo test --no-default-features -- feature_sweeps`. The real-model +example is `examples/experiments/capital-feature-sweep.toml` (run +`capital-clean.toml` first, and convert the SAE as its header shows). + +## 6. Close-out + +- [x] Index row updated with this record. +- [ ] Primary and backup reviewed the evidence. (No backup is assigned.) +- [x] Governing docs/contracts updated, or an issue links the needed update. +- [x] Follow-ups are visible and have owners. +- [x] No model weights, private data, or generated tensor payloads were added. + +**Reviewer notes:** the automated review found the three required problems +and the two observations above, all fixed before merge. It confirmed that +sweeps without `features` keep their files and hash. diff --git a/docs/audits/README.md b/docs/audits/README.md index 1f9c662..35983ad 100644 --- a/docs/audits/README.md +++ b/docs/audits/README.md @@ -23,7 +23,7 @@ backup, fixing that is an audit finding. | K-quant matmul | runtime maintainer / SIMD reviewer | [`crates/ember-core/src/k_quant_matmul.rs`](../../crates/ember-core/src/k_quant_matmul.rs), [`crates/ember-core/src/k_matmul.rs`](../../crates/ember-core/src/k_matmul.rs) | [`k_quant_matmul` tests](../../crates/ember-core/src/k_quant_matmul.rs), pinned known-answer gate | Add the next dated record; see [`CONTRIBUTING.md`](../../CONTRIBUTING.md) | | Loader, quantization, and model families | runtime maintainer / architecture reviewer | [`crates/ember-core/src/loader.rs`](../../crates/ember-core/src/loader.rs), [`crates/ember-core/src/quant_k.rs`](../../crates/ember-core/src/quant_k.rs), [`docs/models.md`](../models.md) | loader/unit tests plus the relevant golden-logit or parity check | Add the next dated record | | Execution plan, hooks, and experiments | runtime maintainer / research reviewer | [`crates/ember-core/src/plan.rs`](../../crates/ember-core/src/plan.rs), [`crates/ember-core/src/planned_decode.rs`](../../crates/ember-core/src/planned_decode.rs), [`crates/ember-core/src/experiments/`](../../crates/ember-core/src/experiments/), [`src/experiments/`](../../src/experiments/) | plan/hook tests, reference-vs-planned parity, capture/intervention contract | [2026-10-21 attention heads](2026-10-21-attention-heads.md) | -| Bundles, reproducibility, and agent traces | research maintainer / release reviewer | [`src/v05/`](../../src/v05/), [`src/agent/`](../../src/agent/), [`docs/reproducibility.md`](../reproducibility.md) | verify/reproduce a small fixture or contract test; inspect provenance | [2026-10-11 sweeps](2026-10-11-sweeps.md), [2026-10-21 head sweeps](2026-10-21-head-sweeps.md), [2026-10-22 feature dictionaries](2026-10-22-feature-dictionaries.md) | +| Bundles, reproducibility, and agent traces | research maintainer / release reviewer | [`src/v05/`](../../src/v05/), [`src/agent/`](../../src/agent/), [`docs/reproducibility.md`](../reproducibility.md) | verify/reproduce a small fixture or contract test; inspect provenance | [2026-10-11 sweeps](2026-10-11-sweeps.md), [2026-10-21 head sweeps](2026-10-21-head-sweeps.md), [2026-10-22 feature dictionaries](2026-10-22-feature-dictionaries.md), [2026-10-22 feature sweeps](2026-10-22-feature-sweeps.md) | | GUI, multimodal, audio, and TTS | product/runtime maintainer / headless reviewer | [`src/gui.rs`](../../src/gui.rs), [`src/multimodal/`](../../src/multimodal/), [`src/tts/`](../../src/tts/) | default-feature and `--no-default-features` builds; applicable focused tests | Add the next dated record | | Python research and probe pipeline | research maintainer / statistics reviewer | [`probes/`](../../probes/), [`tests/`](../../tests/), [`docs/research.md`](../research.md) | compile/smoke plus the relevant pytest and artifact checks | Add the next dated record | | CI, release, and documentation | release maintainer / docs reviewer | [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml), [`scripts/check_docs.py`](../../scripts/check_docs.py) | CI-equivalent gates and docs check | Add the next dated record | diff --git a/docs/experiment-schema-v1.md b/docs/experiment-schema-v1.md index 4173618..47ac923 100644 --- a/docs/experiment-schema-v1.md +++ b/docs/experiment-schema-v1.md @@ -84,10 +84,10 @@ settings are identical whether a default is omitted or explicitly supplied. ## Sweeps An optional `[sweep]` table turns the spec into a template for a layer -sweep (`layers`, optional `positions`, optional `heads`, optional `alphas`, -optional `interventions`, optional `move_bundle_sources`, optional -`[sweep.effect]`; `layers` may be omitted when `heads` or `alphas` is -given); see `docs/experiments.md#layer-sweeps`. Unknown `[sweep]` fields fail, and a +sweep (`layers`, optional `positions`, optional `heads`, optional +`features`, optional `alphas`, optional `interventions`, optional +`move_bundle_sources`, optional `[sweep.effect]`; `layers` may be omitted +when `heads`, `features` or `alphas` is given); see `docs/experiments.md#layer-sweeps`. Unknown `[sweep]` fields fail, and a spec with `[sweep]` is only accepted as a sweep. ## Attribution diff --git a/docs/experiments.md b/docs/experiments.md index 02d245a..e030dbb 100644 --- a/docs/experiments.md +++ b/docs/experiments.md @@ -216,6 +216,7 @@ an ordinary spec (see `examples/experiments/morphology-layer-sweep.toml`): layers = "all" # or [2, 4, 8], or { start = 0, end = 16, step = 2 } positions = [3, 7] # optional: absolute token positions heads = "all" # optional: one attention head per point (attention-heads only) +# features = { dictionary = "l10", ids = [3756, 8633] } # optional: one feature per point; not with layers, heads or positions alphas = [0.0, 2.0, 4.0] # optional: sets `alpha` of the swept interventions interventions = ["replace"] # optional: which interventions move (default: all) ``` @@ -247,6 +248,27 @@ before it runs. A layers x heads grid makes one bundle per point, so a sweep of 2 layers of a 32-head model writes 65 bundles. +`features = { dictionary, ids }` sweeps the features of a dictionary (see +[feature dictionaries](#feature-dictionaries)): each point gives the swept +interventions `features = { dictionary, ids = [F] }`, so each acts on one +feature. `ids` is a list or a `{ start, end, step }` range; there is no +`"all"`, because a dictionary can hold tens of thousands of features and a +sweep writes one bundle per point. A dictionary is trained at one site and +layer, so the swept interventions must be at that site with `layers = +[]`, must not declare `features` themselves, and the sweep cannot +also move layers, heads or positions; it can cross with `alphas`. The swept +interventions must use `zero`, `scale`, `replace`, `add-delta` or +`interpolate`, as for any feature intervention. A sweep sets at most 65,536 +features, and the run checks every id against the dictionary's width before +the first point runs. Point ids are `feature-03756`. `sweep.json` records +`features` (the dictionary and the ids) and each point's `feature`, and +`sweep.csv` and `sweep-effect.csv` gain a `feature` column for feature +sweeps only. See +`examples/experiments/capital-feature-sweep.toml`: eight country features +across eight capital prompts. Each feature has its largest effect on its own +country's prompt. Of the 56 other entries, 50 are exactly 0; the example +header explains the six that are not. + Each *point* is the spec with the swept interventions at `layers = [L]` (and, with `positions`, `tokens = { kind = "absolute-token", index = P }`); everything else is unchanged. The *baseline* is the spec without diff --git a/src/cli_experiment_dictionary.rs b/src/cli_experiment_dictionary.rs index c24f632..d7800f6 100644 --- a/src/cli_experiment_dictionary.rs +++ b/src/cli_experiment_dictionary.rs @@ -69,7 +69,7 @@ pub(crate) fn load_dictionaries( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use crate::cli_experiment::{execute_prepared, prepare_run, PreparedRun}; use crate::experiment_testutil::{ copy_dir, resolve, resolve_err, spec_text, tiny_model, TinyModel, @@ -99,7 +99,7 @@ text = "w8 w1 w33" /// A JumpReLU dictionary for the tiny model's residual stream, written as /// a SAELens folder. Returns the folder and the weights' SHA-256. - fn write_dictionary(dir: &Path, d_in: usize) -> (PathBuf, String) { + pub(crate) fn write_dictionary(dir: &Path, d_in: usize) -> (PathBuf, String) { let folder = dir.join("dictionary"); std::fs::create_dir_all(&folder).unwrap(); let w_enc: Vec = (0..d_in * D_SAE) diff --git a/src/cli_experiment_sweep.rs b/src/cli_experiment_sweep.rs index e1c59b2..7bf5c9d 100644 --- a/src/cli_experiment_sweep.rs +++ b/src/cli_experiment_sweep.rs @@ -73,6 +73,7 @@ pub(crate) fn run_validate_sweep(command: &ValidateArgs, text: &str) -> anyhow:: "positions": definition.positions, "alphas": definition.alphas, "heads": definition.heads, + "features": definition.features, "move_bundle_sources": definition.move_bundle_sources, "effect": definition.effect.as_ref().map(|effect| serde_json::json!({ "capture": effect.capture, @@ -112,6 +113,13 @@ pub(crate) fn run_validate_sweep(command: &ValidateArgs, text: &str) -> anyhow:: serde_json::to_value(heads)? ); } + if let Some(features) = &definition.features { + println!( + " features: {} of dictionary '{}' (checked against its width at run time)", + features.ids.len(), + features.dictionary + ); + } if definition.move_bundle_sources { println!(" bundle sources: read at each point's layer (move_bundle_sources)"); } @@ -179,6 +187,20 @@ pub(crate) fn run_sweep( prepared.model.config.head_dim, ) .map_err(anyhow::Error::msg)?; + // Check the swept feature ids against the dictionary's width before + // anything runs. The points reuse the loaded weights from the cache. + if let Some(features) = &definition.features { + let dictionaries = + crate::cli_experiment_dictionary::load_dictionaries(&prepared, &template)?; + let index = template + .dictionaries + .iter() + .position(|dictionary| dictionary.id == features.dictionary) + .expect("the sweep parse checked the dictionary id"); + dictionaries[index] + .check_ids(&features.ids) + .map_err(|error| anyhow::anyhow!("sweep.features: {error}"))?; + } // Resolve the effect tokens before anything runs, so a target that is // not one token fails without a wasted sweep. let effect = definition @@ -261,6 +283,7 @@ pub(crate) fn run_sweep( position: point.position, alpha: point.alpha, head: point.head, + feature: point.feature, bundle: reference(outcome, &point.relative_dir), inputs, effect: point_effect, @@ -286,6 +309,7 @@ pub(crate) fn run_sweep( .transpose() .map_err(|error| anyhow::anyhow!("{error}"))?, head_count: definition.heads.is_some().then_some(n_heads), + features: definition.features.clone(), interventions: definition.interventions.clone(), effect, baseline: reference(&base, &baseline.relative_dir), @@ -399,6 +423,12 @@ fn print_table(manifest: &SweepManifest) { if let Some(heads) = &manifest.heads { axes.push(format!("heads {heads:?}")); } + if let Some(features) = &manifest.features { + axes.push(format!( + "features {:?} of dictionary {:?}", + features.ids, features.dictionary + )); + } if let Some(alphas) = &manifest.alphas { axes.push(format!("alphas {alphas:?}")); } @@ -1245,6 +1275,113 @@ foil = 13 body } + #[test] + fn feature_sweeps_run_one_feature_per_point_and_verify() { + let model = tiny_model("sweep-feature-grid", 3, 64, false); + let (folder, sha) = + crate::cli_experiment_dictionary::tests::write_dictionary(&model.dir, 64); + let config_sha = ember::v05::manifest::sha256_hex( + &std::fs::read(folder.join(ember::v05::dictionary::CONFIG_FILE)).unwrap(), + ); + let dictionary = format!( + "\n[[dictionaries]]\nid = \"d\"\npath = \"{}\"\nsha256 = \"{sha}\"\n\ + config_sha256 = \"{config_sha}\"\nsite = \"residual-post-mlp\"\nlayer = 1\n", + folder.display() + ); + let body = format!( + "{}{dictionary}", + EFFECT_BODY.replace("layers = [0]", "layers = [1]") + ); + let table = EFFECT_TABLE.replace( + "[sweep]\nlayers = \"all\"\n", + "[sweep]\nfeatures = { dictionary = \"d\", ids = [5, 0, 9] }\n", + ); + let text = format!("{}{table}", spec_text(&model, "reference", 2, &body)); + let spec = model.dir.join("sweep.toml"); + std::fs::write(&spec, &text).unwrap(); + let out = model.dir.join("sweep-out"); + run_sweep( + &run_args(&spec, Some(out.clone())), + &text, + KStrategy::Auto, + false, + ) + .unwrap(); + let report = verify_sweep(&out, &VerifyOptions::default()).unwrap(); + assert!(report.ok, "{:?}", report.checks); + assert_eq!(report.bundles, 4); + let manifest = ember::v05::sweep::read_manifest(&out).unwrap(); + let features = manifest.features.clone().unwrap(); + assert_eq!( + (features.dictionary.as_str(), features.ids.clone()), + ("d", vec![0, 5, 9]) + ); + assert_eq!(manifest.layers, Vec::::new()); + assert_eq!( + manifest + .points + .iter() + .map(|p| (p.id.as_str(), p.feature)) + .collect::>(), + [ + ("feature-00", Some(0)), + ("feature-05", Some(5)), + ("feature-09", Some(9)) + ] + ); + let csv = std::fs::read_to_string(out.join(SWEEP_CSV_FILE)).unwrap(); + assert!( + csv.starts_with("point,layer,position,feature,input_id,"), + "{csv}" + ); + let table = std::fs::read_to_string(out.join(SWEEP_EFFECT_CSV_FILE)).unwrap(); + assert!( + table.starts_with("point,layer,position,alpha,feature,n,"), + "{table}" + ); + assert!(table.contains("\nfeature-05,,,,5,3,"), "{table}"); + + // A forged feature fails verification. + let reseal = |manifest: &SweepManifest| { + std::fs::write( + out.join(SWEEP_MANIFEST_FILE), + serde_json::to_vec(manifest).unwrap(), + ) + .unwrap(); + verify_sweep(&out, &VerifyOptions::default()).unwrap() + }; + let failed = |report: &SweepVerification, name: &str| { + report.checks.iter().any(|c| !c.ok && c.name == name) + }; + let mut forged = manifest.clone(); + forged.points[1].feature = Some(6); + forged.sweep_hash = forged.compute_hash(); + assert!(failed(&reseal(&forged), "point feature-05 coordinates")); + let mut forged = manifest.clone(); + forged.features.as_mut().unwrap().ids = vec![0, 5, 8]; + forged.sweep_hash = forged.compute_hash(); + assert!(failed(&reseal(&forged), "sweep definition")); + + // A feature id past the dictionary's width fails before any point + // runs. + let wide = text.replace("ids = [5, 0, 9]", "ids = [5, 16]"); + assert_ne!(wide, text); + let wide_out = model.dir.join("sweep-wide"); + let error = run_sweep( + &run_args(&spec, Some(wide_out.clone())), + &wide, + KStrategy::Auto, + false, + ) + .unwrap_err() + .to_string(); + assert!( + error.contains("sweep.features: feature 16 is out of range"), + "{error}" + ); + assert!(!wide_out.exists(), "no bundle is written"); + } + #[test] fn head_sweeps_run_every_head_and_verify() { let model = tiny_model("sweep-head-grid", 3, 64, true); diff --git a/src/v05/sweep.rs b/src/v05/sweep.rs index 78c3670..e852902 100644 --- a/src/v05/sweep.rs +++ b/src/v05/sweep.rs @@ -69,6 +69,9 @@ pub const SWEEP_SPEC_FILE: &str = "sweep.toml"; pub const SWEEP_MANIFEST_FILE: &str = "sweep.json"; pub const SWEEP_CSV_FILE: &str = "sweep.csv"; pub const SWEEP_RUNTIME_FILE: &str = "sweep-runtime.json"; +/// The most features one sweep may set, one point each: the width of the +/// largest dictionary that `MAX_WEIGHTS_BYTES` admits. +pub const MAX_SWEPT_FEATURES: usize = 65_536; /// The `[sweep]` table. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -89,6 +92,11 @@ pub struct RawSweepSpec { /// one per point: `"all"`, a list, or a range. #[serde(default, skip_serializing_if = "Option::is_none")] pub heads: Option, + /// Features of a dictionary (ADR-0003) the swept interventions act on, + /// one per point: each point gives them `features = { dictionary, ids = + /// [F] }`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub features: Option, /// Intervention ids to sweep (default: every intervention). #[serde(default, skip_serializing_if = "Option::is_none")] pub interventions: Option>, @@ -101,6 +109,26 @@ pub struct RawSweepSpec { pub effect: Option, } +/// `[sweep] features`: a dictionary and the feature ids to sweep, as a list +/// or a `{ start, end, step }` range (end exclusive). There is no `"all"`: a +/// dictionary can hold tens of thousands of features, one bundle each. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawFeatureSweep { + pub dictionary: String, + pub ids: LayerSelector, +} + +/// The features a sweep moves its interventions through (`sweep.json` +/// records it as it is). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FeatureSweep { + pub dictionary: String, + /// Ascending, without repeats. + pub ids: Vec, +} + /// A parsed and validated sweep specification. #[derive(Debug, Clone)] pub struct SweepDefinition { @@ -113,6 +141,8 @@ pub struct SweepDefinition { pub positions: Option>, /// `None`: the swept interventions keep their declared heads. pub heads: Option, + /// `None`: the swept interventions act on whole rows, as declared. + pub features: Option, /// Swept `alpha` values, ascending. pub alphas: Option>, /// Swept intervention ids, in declaration order. @@ -135,6 +165,7 @@ pub struct DerivedSpec { pub id: String, pub layer: Option, pub head: Option, + pub feature: Option, pub position: Option, pub alpha: Option, /// The derived spec text (becomes the bundle's `experiment.toml`). @@ -227,12 +258,20 @@ impl SweepDefinition { .collect() } }; - if sweep.layers.is_none() && sweep.heads.is_none() && sweep.alphas.is_none() { + if sweep.layers.is_none() + && sweep.heads.is_none() + && sweep.features.is_none() + && sweep.alphas.is_none() + { return Err(SpecError::at( "sweep", - "a sweep needs `layers`, `heads`, `alphas`, or a combination", + "a sweep needs `layers`, `heads`, `features`, `alphas`, or a combination", )); } + let features = match &sweep.features { + None => None, + Some(raw) => Some(Self::parse_features(raw, &sweep, &template, &swept)?), + }; if sweep.layers.is_none() && sweep.positions.is_some() { return Err(SpecError::at( "sweep.positions", @@ -418,6 +457,7 @@ impl SweepDefinition { layers: sweep.layers, positions, heads: sweep.heads, + features, alphas, interventions: swept, move_bundle_sources, @@ -427,6 +467,134 @@ impl SweepDefinition { }) } + /// Check `[sweep] features` against the template: a dictionary is + /// trained at one site and layer, so the swept interventions stay there + /// and the sweep moves nothing else. + fn parse_features( + raw: &RawFeatureSweep, + sweep: &RawSweepSpec, + template: &ExperimentSpecV1, + swept: &[String], + ) -> Result { + if sweep.layers.is_some() || sweep.heads.is_some() || sweep.positions.is_some() { + return Err(SpecError::at( + "sweep.features", + "a feature sweep keeps the swept interventions at the dictionary's site and \ + layer; it cannot also sweep layers, heads or positions", + )); + } + let dictionary = template + .dictionaries + .iter() + .find(|dictionary| dictionary.id == raw.dictionary) + .ok_or_else(|| { + SpecError::at( + "sweep.features.dictionary", + format!("no [[dictionaries]] entry has id {:?}", raw.dictionary), + ) + })?; + let ids = match &raw.ids { + LayerSelector::All(_) => { + return Err(SpecError::at( + "sweep.features.ids", + "list the feature ids, or give a { start, end, step } range: a sweep of \ + every feature would write one bundle per feature", + )) + } + LayerSelector::List(list) => { + if list.is_empty() { + return Err(SpecError::at( + "sweep.features.ids", + "the feature list is empty", + )); + } + let mut list = list.clone(); + list.sort_unstable(); + list.dedup(); + list + } + LayerSelector::Range(range) => { + if range.step == 0 || range.start >= range.end { + return Err(SpecError::at( + "sweep.features.ids", + "a feature range needs start < end and step >= 1", + )); + } + // Count the range before allocating it: an end near + // usize::MAX must fail here, not exhaust memory. + let count = (range.end - range.start - 1) / range.step + 1; + if count > MAX_SWEPT_FEATURES { + return Err(SpecError::at( + "sweep.features.ids", + format!( + "the range selects {count} features; a sweep sets at most \ + {MAX_SWEPT_FEATURES}" + ), + )); + } + (range.start..range.end).step_by(range.step).collect() + } + }; + if ids.len() > MAX_SWEPT_FEATURES { + return Err(SpecError::at( + "sweep.features.ids", + format!( + "the list names {} features; a sweep sets at most {MAX_SWEPT_FEATURES}", + ids.len() + ), + )); + } + for (index, intervention) in template.interventions.iter().enumerate() { + if !swept.contains(&intervention.id) { + continue; + } + let path = format!("interventions[{index}]"); + if intervention.features.is_some() { + return Err(SpecError::at( + format!("{path}.features"), + "a feature sweep sets features at each point; remove features here", + )); + } + use crate::v05::intervention::InterventionOperation as Op; + if matches!( + intervention.operation, + Op::Steer { .. } | Op::AblateProjection | Op::RestoreOriginal + ) { + return Err(SpecError::at( + "sweep.features", + format!( + "intervention '{}': {} does not act on features; use zero, scale, \ + replace, add-delta or interpolate", + intervention.id, + intervention.operation.kind_name() + ), + )); + } + let at_dictionary = intervention.site == dictionary.site + && match &intervention.layers { + LayerSelector::List(layers) => layers.as_slice() == [dictionary.layer], + LayerSelector::Range(range) => { + range.start == dictionary.layer && range.end == dictionary.layer + 1 + } + LayerSelector::All(_) => false, + }; + if !at_dictionary { + return Err(SpecError::at( + path, + format!( + "a feature sweep acts at dictionary {:?}'s site and layer: set site = \ + \"{}\" and layers = [{}]", + dictionary.id, dictionary.site, dictionary.layer + ), + )); + } + } + Ok(FeatureSweep { + dictionary: raw.dictionary.clone(), + ids, + }) + } + /// The layers of this sweep for an `n_layers`-layer model (empty when /// the sweep does not move layers). pub fn resolve_layers(&self, n_layers: usize) -> Result, SpecError> { @@ -504,10 +672,15 @@ impl SweepDefinition { Some(_) => self.resolve_heads(n_heads)?.len(), None => 1, }; + let features = self + .features + .as_ref() + .map_or(1, |features| features.ids.len()); let positions = self.positions.as_ref().map_or(1, Vec::len); let alphas = self.alphas.as_ref().map_or(1, Vec::len); Ok(layers .checked_mul(heads) + .and_then(|count| count.checked_mul(features)) .and_then(|count| count.checked_mul(positions)) .and_then(|count| count.checked_mul(alphas))) } @@ -606,6 +779,7 @@ impl SweepDefinition { id: id.to_string(), layer: None, head: None, + feature: None, position: None, alpha: None, text, @@ -627,7 +801,7 @@ impl SweepDefinition { } /// Every point for an `n_layers`-layer, `n_heads`-head model: layers - /// ascending, then heads, then positions, then alphas. + /// ascending, then heads, then features, then positions, then alphas. pub fn points(&self, n_layers: usize, n_heads: usize) -> Result, SpecError> { let resolved_layers = self.resolve_layers(n_layers)?; let layers: Vec> = if self.layers.is_some() { @@ -641,6 +815,15 @@ impl SweepDefinition { vec![None] }; let head_width = n_heads.saturating_sub(1).to_string().len().max(2); + let features: Vec> = match &self.features { + None => vec![None], + Some(features) => features.ids.iter().copied().map(Some).collect(), + }; + let feature_width = self + .features + .as_ref() + .and_then(|features| features.ids.last()) + .map_or(2, |&last| last.to_string().len().max(2)); let alphas: Vec> = match &self.alphas { None => vec![None], Some(list) => list.iter().copied().map(Some).collect(), @@ -678,12 +861,16 @@ impl SweepDefinition { None => vec![None], Some(list) => list.iter().copied().map(Some).collect(), }; - let mut out = - Vec::with_capacity(layers.len() * heads.len() * positions.len() * alphas.len()); - let grid = layers - .iter() - .flat_map(|&layer| heads.iter().map(move |&head| (layer, head))); - for (layer, head) in grid { + let mut out = Vec::with_capacity( + layers.len() * heads.len() * features.len() * positions.len() * alphas.len(), + ); + let grid = layers.iter().flat_map(|&layer| { + heads.iter().flat_map({ + let features = &features; + move |&head| features.iter().map(move |&feature| (layer, head, feature)) + }) + }); + for (layer, head, feature) in grid { for &position in &positions { for &alpha in &alphas { let mut parts = Vec::new(); @@ -693,6 +880,9 @@ impl SweepDefinition { if let Some(head) = head { parts.push(format!("head-{head:0head_width$}")); } + if let Some(feature) = feature { + parts.push(format!("feature-{feature:0feature_width$}")); + } if let Some(position) = position { parts.push(format!("pos-{position}")); } @@ -736,6 +926,18 @@ impl SweepDefinition { toml::Value::Array(vec![toml::Value::Integer(head as i64)]), ); } + if let (Some(feature), Some(features)) = (feature, &self.features) { + let mut selection = toml::Table::new(); + selection.insert( + "dictionary".into(), + toml::Value::String(features.dictionary.clone()), + ); + selection.insert( + "ids".into(), + toml::Value::Array(vec![toml::Value::Integer(feature as i64)]), + ); + table.insert("features".into(), toml::Value::Table(selection)); + } if let (Some(alpha), Some(toml::Value::Table(operation))) = (alpha, table.get_mut("operation")) { @@ -757,6 +959,9 @@ impl SweepDefinition { if let Some(head) = head { coordinates.push(format!("head {head}")); } + if let Some(feature) = feature { + coordinates.push(format!("feature {feature}")); + } if let Some(position) = position { coordinates.push(format!("token {position}")); } @@ -770,6 +975,7 @@ impl SweepDefinition { let mut derived = self.derive(&id, &what, document, format!("points/{id}"))?; derived.layer = layer; derived.head = head; + derived.feature = feature; derived.position = position; derived.alpha = alpha; out.push(derived); @@ -838,6 +1044,9 @@ pub struct SweepPointRecord { /// The point's head, for head sweeps. #[serde(default, skip_serializing_if = "Option::is_none")] pub head: Option, + /// The point's feature, for feature sweeps. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub feature: Option, #[serde(flatten)] pub bundle: SweepBundleRef, pub inputs: Vec, @@ -867,6 +1076,9 @@ pub struct SweepManifest { pub heads: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub head_count: Option, + /// For feature sweeps: the dictionary and the swept feature ids. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub features: Option, pub interventions: Vec, /// With `[sweep.effect]`: the metric, interval settings and resolved /// target/foil tokens. @@ -895,8 +1107,10 @@ impl SweepManifest { // The alpha column exists only for alpha sweeps, so earlier sweeps // keep their exact table. let with_alpha = self.alphas.is_some(); - // The head column likewise exists only for head sweeps. + // The head column likewise exists only for head sweeps, and the + // feature column only for feature sweeps. let with_head = self.heads.is_some(); + let with_feature = self.features.is_some(); let mut out = String::from(if with_alpha { "point,layer,position,alpha," } else { @@ -905,6 +1119,9 @@ impl SweepManifest { if with_head { out.push_str("head,"); } + if with_feature { + out.push_str("feature,"); + } out.push_str( "input_id,first_divergent_step,generated_text_equal,\ peak_relative_l2,peak_capture_id,peak_site,peak_layer,captures_exact,\ @@ -927,6 +1144,9 @@ impl SweepManifest { if with_head { alpha.push_str(&format!("{},", opt(&point.head))); } + if with_feature { + alpha.push_str(&format!("{},", opt(&point.feature))); + } let effect = if with_effect { format!( ",{},{},{}", @@ -964,14 +1184,18 @@ impl SweepManifest { value.as_ref().map(ToString::to_string).unwrap_or_default() } self.effect.as_ref()?; - // A head column only for head sweeps, so earlier tables stay as they - // were. + // A head column only for head sweeps and a feature column only for + // feature sweeps, so earlier tables stay as they were. let with_head = self.heads.is_some(); + let with_feature = self.features.is_some(); let mut out = String::from(if with_head { "point,layer,position,alpha,head," } else { "point,layer,position,alpha," }); + if with_feature { + out.push_str("feature,"); + } out.push_str( "n,mean,sd,standard_error,ci_low,ci_high,positive,negative,zero,sign_test_p\n", ); @@ -979,11 +1203,14 @@ impl SweepManifest { let Some(effect) = &point.effect else { continue; }; - let head = if with_head { + let mut head = if with_head { format!("{},", opt(&point.head)) } else { String::new() }; + if with_feature { + head.push_str(&format!("{},", opt(&point.feature))); + } out.push_str(&format!( "{},{},{},{},{head}{},{},{},{},{},{},{},{},{},{}\n", point.id, @@ -1291,11 +1518,12 @@ fn verify_and_load_sweep( definition.name == manifest.experiment && layers == manifest.layers && heads == manifest.heads + && definition.features == manifest.features && definition.positions == manifest.positions && definition.alphas == manifest.alphas && definition.interventions == manifest.interventions, - "experiment name, layers, heads, positions, alphas and swept interventions match the \ - spec", + "experiment name, layers, heads, features, positions, alphas and swept interventions \ + match the spec", ); match (&definition.effect, &manifest.effect) { (None, None) => {} @@ -1462,22 +1690,24 @@ fn verify_and_load_sweep( point.text.as_str(), point.layer, point.head, + point.feature, point.position, point.alpha, ) }); match expected { - Some((text, layer, head, position, alpha)) => { + Some((text, layer, head, feature, position, alpha)) => { bundle_checks(&mut report, &label, &bundle, &record.bundle, text); report.check( format!("{label} coordinates"), layer == record.layer && head == record.head + && feature == record.feature && position == record.position && alpha.map(f64::to_bits) == record.alpha.map(f64::to_bits), format!( - "layer {:?} head {:?} position {:?} alpha {:?}", - record.layer, record.head, record.position, record.alpha + "layer {:?} head {:?} feature {:?} position {:?} alpha {:?}", + record.layer, record.head, record.feature, record.position, record.alpha ), ); } @@ -1789,6 +2019,118 @@ interventions = ["replace"] ) } + /// SPEC with a dictionary at the swept intervention's site and layer + /// (residual-post-mlp, layer 7) and a feature sweep in place of the + /// layer sweep. + fn feature_spec(sweep: &str) -> String { + let hash = "0".repeat(64); + SPEC.replace( + "[output]\n", + &format!( + "[[dictionaries]]\nid = \"d\"\npath = \"dict\"\nsha256 = \"{hash}\"\n\ + config_sha256 = \"{hash}\"\nsite = \"residual-post-mlp\"\nlayer = 7\n\n[output]\n" + ), + ) + .replace( + "[sweep]\nlayers = { start = 0, end = 12, step = 4 }\ninterventions = [\"replace\"]\n", + &format!("[sweep]\n{sweep}interventions = [\"replace\"]\n"), + ) + } + + #[test] + fn feature_sweeps_set_one_feature_per_point() { + let text = feature_spec("features = { dictionary = \"d\", ids = [40, 3, 40, 12] }\n"); + let definition = SweepDefinition::parse(&text).unwrap(); + let features = definition.features.clone().unwrap(); + assert_eq!(features.ids, vec![3, 12, 40], "ascending, without repeats"); + assert_eq!(definition.point_count(12, 4).unwrap(), Some(3)); + let points = definition.points(12, 4).unwrap(); + let ids: Vec<&str> = points.iter().map(|p| p.id.as_str()).collect(); + assert_eq!(ids, vec!["feature-03", "feature-12", "feature-40"]); + let point = &points[1]; + assert_eq!(point.feature, Some(12)); + assert_eq!(point.layer, None); + let find = |id: &str| { + point + .resolved + .interventions + .iter() + .find(|i| i.id == id) + .unwrap() + .clone() + }; + let swept = find("replace"); + assert_eq!( + swept.features, + Some(crate::v05::dictionary::FeatureSelection { + dictionary: "d".into(), + ids: vec![12], + }) + ); + assert_eq!( + swept.layers, + LayerSelector::List(vec![7]), + "it stays at the dictionary's layer" + ); + assert_eq!( + find("fixed").features, + None, + "an unswept intervention is unchanged" + ); + + let range = feature_spec( + "features = { dictionary = \"d\", ids = { start = 0, end = 10, step = 5 } }\n", + ); + let range = SweepDefinition::parse(&range).unwrap(); + assert_eq!(range.features.unwrap().ids, vec![0, 5]); + } + + #[test] + fn feature_sweeps_refuse_what_a_dictionary_cannot_do() { + let refuse = |text: &str| SweepDefinition::parse(text).unwrap_err().to_string(); + let features = "features = { dictionary = \"d\", ids = [1] }\n"; + assert!(refuse(&feature_spec( + "features = { dictionary = \"d\", ids = \"all\" }\n" + )) + .contains("list the feature ids")); + assert!(refuse(&feature_spec( + "features = { dictionary = \"d\", ids = [] }\n" + )) + .contains("empty")); + assert!(refuse(&feature_spec(&format!("layers = [7]\n{features}"))) + .contains("cannot also sweep")); + assert!(refuse(&feature_spec( + "features = { dictionary = \"x\", ids = [1] }\n" + )) + .contains("no [[dictionaries]] entry")); + let declared = feature_spec(features).replace( + "source = { kind = \"capture-from-current-run\", capture_id = \"rows\" }", + "source = { kind = \"capture-from-current-run\", capture_id = \"rows\" }\n\ + features = { dictionary = \"d\", ids = [2] }", + ); + assert!(refuse(&declared).contains("remove features here")); + let elsewhere = feature_spec(features).replacen("layers = [7]", "layers = [3]", 1); + assert!(refuse(&elsewhere).contains("layers = [7]")); + // A huge range fails before it allocates. + let huge = format!( + "features = {{ dictionary = \"d\", ids = {{ start = 0, end = {}, step = 1 }} }}\n", + i64::MAX + ); + assert!(refuse(&feature_spec(&huge)).contains("a sweep sets at most 65536")); + let ceiling = + "features = { dictionary = \"d\", ids = { start = 0, end = 65536, step = 1 } }\n"; + assert!(SweepDefinition::parse(&feature_spec(ceiling)).is_ok()); + let steer = feature_spec(features).replace( + "operation = { kind = \"replace\" }\nsource = { kind = \"capture-from-current-run\", \ + capture_id = \"rows\" }", + "operation = { kind = \"steer\", alpha = 2.0 }\n\ + source = { kind = \"inline-vector\", values = [1.0] }", + ); + assert_ne!(steer, feature_spec(features)); + let error = refuse(&steer); + assert!(error.contains("steer does not act on features"), "{error}"); + } + #[test] fn head_sweeps_set_one_head_per_point() { let text = head_spec("layers = [3, 7]\nheads = \"all\"\n"); @@ -1979,12 +2321,14 @@ interventions = ["replace"] alphas: None, heads: None, head_count: None, + features: None, points: vec![SweepPointRecord { id: "layer-01".into(), layer: Some(1), position: None, alpha: None, head: None, + feature: None, bundle: SweepBundleRef { bundle: "points/layer-01".into(), ..reference From 169ff726b05715a779c3bc9d6dda4048c399e8d8 Mon Sep 17 00:00:00 2001 From: voidwest Date: Thu, 22 Oct 2026 22:28:28 +0300 Subject: [PATCH 2/2] docs(examples): sweep eight country features across the capital prompts capital-feature-sweep.toml patches each of the eight country features into all eight corrupted capital prompts. Each feature has its largest effect on its own country's prompt. Of the 56 other entries, 50 are exactly 0. The header explains the six that are not, from a full feature capture of the rows. Co-Authored-By: Claude Opus 5.5 --- examples/experiments/README.md | 6 + .../experiments/capital-feature-sweep.toml | 259 ++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 examples/experiments/capital-feature-sweep.toml diff --git a/examples/experiments/README.md b/examples/experiments/README.md index 6300a55..a3a18c7 100644 --- a/examples/experiments/README.md +++ b/examples/experiments/README.md @@ -102,6 +102,12 @@ copy pinned to `unsloth/Llama-3.2-1B-Instruct` revision `scripts/convert_linear_sae.py` step. Run `capital-clean.toml` before the sweep. +12. `capital-feature-sweep.toml` — a feature sweep (`[sweep] features`): each + of the eight country features, patched into all eight corrupted prompts. + Each feature has its largest effect on its own country's prompt. Of the + 56 other entries, 50 are exactly 0; the header explains the six that are + not. Needs the same SAE and `capital-clean.toml`. + ## Commands ```bash diff --git a/examples/experiments/capital-feature-sweep.toml b/examples/experiments/capital-feature-sweep.toml new file mode 100644 index 0000000..e4f3469 --- /dev/null +++ b/examples/experiments/capital-feature-sweep.toml @@ -0,0 +1,259 @@ +# A feature sweep (`[sweep] features`): each point patches one of the eight +# country features from capital-feature-capture.toml into every corrupted +# capital prompt, from that prompt's own clean run, entering block 10. The +# interventions declare no features; the sweep gives them one per point. The +# source rows are runs/capital-clean (capital-clean.toml; run it first). +# +# The dictionary is a sparse autoencoder for Llama-3.2-1B-Instruct, trained +# on the output of block 9 (Apache-2.0, 32,768 features). That output is the +# input of block 10, so it is bound to residual-pre-attention layer 10. It is +# a PyTorch state dict, not a SAELens folder; convert it once: +# +# curl -L -o qresearch-l9.pt https://huggingface.co/qresearch/Llama-3.2-1B-Instruct-SAE-l9/resolve/4fd505efade04b357f98666f69bae5fd718c039c/Llama-3.2-1B-Instruct-SAE-l9.pt +# # sha256 d89b1b4953379a824f1e4a31599ac42c66eb8dc3382b7d960dce67d97ebc7ace +# python scripts/convert_linear_sae.py qresearch-l9.pt \ +# dictionaries/qresearch-llama-3.2-1b-instruct-sae-l9 \ +# --model-name meta-llama/Llama-3.2-1B-Instruct \ +# --hook-name blocks.9.hook_resid_post \ +# --source "huggingface.co/qresearch/Llama-3.2-1B-Instruct-SAE-l9@4fd505efade04b357f98666f69bae5fd718c039c" \ +# --license apache-2.0 +# +# The two hashes below are the converter's output for that file. On the +# clean prompts the fit is a mean l0 of 46 active features and a median +# relative reconstruction error of 0.48. +# +# Result on the pinned model (Apple M1 Pro, reference mode, 2026-10-22; one +# deterministic run): the effect on logit(clean capital) - logit(corrupted +# capital) of each point (row) on each corrupted prompt (column, named by its +# clean country; its text names another country): +# +# feature (country) france germany japan egypt poland portugal italy spain +# 3756 (France) 4.69 0.00 0.00 0.00 0.00 0.00 1.12 0.00 +# 8633 (Germany) 0.00 2.72 0.00 0.00 0.36 0.00 0.00 0.06 +# 18838 (Poland) 0.00 0.00 0.00 0.00 1.23 0.00 0.00 0.00 +# 19172 (Spain) 0.00 1.20 0.00 0.00 0.00 0.10 0.00 3.27 +# 21761 (Japan) 0.00 0.00 6.46 0.00 0.00 0.00 0.00 0.00 +# 22634 (Egypt) 0.00 0.00 0.00 1.81 0.00 0.00 0.00 0.00 +# 22778 (Italy) 0.47 0.00 0.00 0.00 0.00 0.00 3.20 0.00 +# 27466 (Portugal) 0.00 0.00 0.00 0.00 0.00 3.76 0.00 0.00 +# +# The diagonal adds the clean country's feature (mean +3.39, as in +# capital-feature-patching-sweep.toml). Five off-diagonal entries take a +# country out of the corrupted text. The "italy" prompt reads "The capital of +# France is", so the clean Italy value of the France feature takes France out +# (+1.12). The same occurs for Germany on "spain", Spain on "germany" and +# Italy on "france". The Germany feature is also active (0.32) on the Austria +# prompt and off on clean Poland, so the patch takes it out (+0.36). One +# entry adds a related country: the Spain feature is weakly active (0.11) on +# the clean Portugal prompt and off on Sweden (+0.10). The other 50 entries +# are exactly 0: those features are off in both rows, so the patch changes +# nothing. +# +# ember experiment run examples/experiments/capital-clean.toml +# ember experiment run examples/experiments/capital-feature-sweep.toml + +schema = "ember.experiment.v1" + +[experiment] +name = "capital-feature-sweep" +description = "Patch each of the eight country features into every corrupted capital prompt, one feature per point." + +[model] +path = "Llama-3.2-1B-Instruct-Q8_0.gguf" +expected_sha256 = "432f310a77f4650a88d0fd59ecdd7cebed8d684bafea53cbff0473542964f0c3" +tokenizer = "tokenizer.json" +tokenizer_expected_sha256 = "6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b" + +[execution] +mode = "reference" +threads = 4 +deterministic = true + +[generation] +max_new_tokens = 0 +temperature = 0.0 + +[[inputs]] +id = "france" +text = "The capital of Italy is the city of" + +[[inputs]] +id = "germany" +text = "The capital of Spain is the city of" + +[[inputs]] +id = "japan" +text = "The capital of Russia is the city of" + +[[inputs]] +id = "egypt" +text = "The capital of Greece is the city of" + +[[inputs]] +id = "poland" +text = "The capital of Austria is the city of" + +[[inputs]] +id = "portugal" +text = "The capital of Sweden is the city of" + +[[inputs]] +id = "italy" +text = "The capital of France is the city of" + +[[inputs]] +id = "spain" +text = "The capital of Germany is the city of" + +[[dictionaries]] +id = "l10" +path = "dictionaries/qresearch-llama-3.2-1b-instruct-sae-l9" +sha256 = "8ade5dc5488d13676a26cad16b4d9bf88c947b9f6df44441996fe87ffff6977f" +config_sha256 = "ac74e31f66b57f8ce784e8741bfec3e7d5ade408b03959e1f465331196cd49f5" +site = "residual-pre-attention" +layer = 10 + +[[captures]] +id = "answer" +site = "logits" + +[captures.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-france" +site = "residual-pre-attention" +layers = [10] +inputs = ["france"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "france", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-germany" +site = "residual-pre-attention" +layers = [10] +inputs = ["germany"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "germany", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-japan" +site = "residual-pre-attention" +layers = [10] +inputs = ["japan"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "japan", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-egypt" +site = "residual-pre-attention" +layers = [10] +inputs = ["egypt"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "egypt", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-poland" +site = "residual-pre-attention" +layers = [10] +inputs = ["poland"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "poland", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-portugal" +site = "residual-pre-attention" +layers = [10] +inputs = ["portugal"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "portugal", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-italy" +site = "residual-pre-attention" +layers = [10] +inputs = ["italy"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "italy", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[[interventions]] +id = "patch-spain" +site = "residual-pre-attention" +layers = [10] +inputs = ["spain"] +operation = { kind = "replace" } +source = { kind = "capture-from-bundle", bundle_path = "runs/capital-clean", capture_id = "final-residual", input_id = "spain", layer = 10 } + +[interventions.tokens] +kind = "prompt-final" + +[output] +directory = "runs/capital-feature-sweep" +overwrite = false + +[sweep] +features = { dictionary = "l10", ids = [3756, 8633, 18838, 19172, 21761, 22634, 22778, 27466] } + +[sweep.effect] +capture = "answer" + +[[sweep.effect.targets]] +input = "france" +target = " Paris" +foil = " Rome" + +[[sweep.effect.targets]] +input = "germany" +target = " Berlin" +foil = " Madrid" + +[[sweep.effect.targets]] +input = "japan" +target = " Tokyo" +foil = " Moscow" + +[[sweep.effect.targets]] +input = "egypt" +target = " Cairo" +foil = " Athens" + +[[sweep.effect.targets]] +input = "poland" +target = " Warsaw" +foil = " Vienna" + +[[sweep.effect.targets]] +input = "portugal" +target = " Lisbon" +foil = " Stockholm" + +[[sweep.effect.targets]] +input = "italy" +target = " Rome" +foil = " Paris" + +[[sweep.effect.targets]] +input = "spain" +target = " Madrid" +foil = " Berlin"