From 0bd8d03ba472813ffd9630388eda50de18663fb6 Mon Sep 17 00:00:00 2001 From: Jen D Date: Sat, 31 Oct 2026 01:53:06 +0800 Subject: [PATCH 1/2] Port research controls, session metadata and better extraction to main These came from research-controls (d022fe3), whose PR #2 was stacked onto fix/steering-review and merged after that branch had already merged, so none of it reached main. Ported onto the renamed modulo package and the features added since: - Controls: SteeringAgent(control="real"|"sham"|"random"). Sham applies nothing; random applies a fixed random direction of the same magnitude. Everything the model sees is identical across conditions; with disclose_controls (any --control) the prompt says, truthfully in every condition, that some sessions are controls. --control assign --blind, replay.py --blind. - Reports record the session's control condition, so sham "it worked" reports can be filtered out (optional field; older reports still valid). - Session logs start with a header (argv, git commit, versions, control, RNG seed, generation settings, rails, vector sha256s); turns record the rendered system prompt and the unsteered NLL of steered outputs. --rng-seed, --temperature, --top-p, --max-new-tokens. - Extraction averages over response tokens by default (split-half cosine on Qwen2.5-7B: sycophancy 0.33 -> 0.83, refusal 0.10 -> 0.68); --read last keeps the old readout. Sycophancy, refusal and creativity get their own question sets; the other starters use a better general mix. - Optional --nll-threshold incoherence check, off by default (NLL doesn't separate word salad from coherent heavy role-play). - CPU tests run in GitHub Actions. Not ported: making auto_extract opt-in. Main has since built on it being on by default (11 starters, only-missing extraction). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 24 ++++++ AGENTS.md | 66 ++++++++++++++- README.md | 4 + agent/loop.py | 105 ++++++++++++++++++++--- extract_contrastive.py | 17 ++-- modulo/agent.py | 107 +++++++++++++++++++---- modulo/cli.py | 7 +- modulo/extraction.py | 163 +++++++++++++++++++++++++++--------- modulo/reports.py | 7 +- replay.py | 33 ++++++-- tests/test_agent_logic.py | 148 +++++++++++++++++++++++++++++++- tests/test_steering_gpu.py | 32 +++++++ 12 files changed, 624 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..893b837 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,24 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + +jobs: + cpu: + # GPU tests (tests/test_steering_gpu.py) skip here; run them on a GPU box + # after touching the steering path. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install CPU-only PyTorch + run: pip install torch --index-url https://download.pytorch.org/whl/cpu + - name: Install dependencies + run: pip install -r requirements.txt + - name: Run tests + run: pytest -q diff --git a/AGENTS.md b/AGENTS.md index 0ec3525..4687e79 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,6 +73,15 @@ python verify_vector.py --vector sycophancy --strength 10 # baseline vs ± Earlier notes said CAA vectors didn't work on Qwen, but those tests ran while steering was silently a no-op during generation, so that needs re-testing. +**Extraction readout:** by default each response's activation is the mean over all its generated tokens (`read="mean"`; `--read` on `extract_contrastive.py` and `modulo extract`). Reading only the last token (`--read last`) is far noisier. On Qwen2.5-7B at layer 14, the cosine between vectors extracted from two disjoint halves of the questions was: + +| vector | last token | response mean | +|---|---|---| +| sycophancy | 0.33 | 0.83 | +| refusal | 0.10 | 0.68 | + +Sycophancy, refusal and creativity have their own question sets (e.g. sycophancy mixes false claims, true claims and the user's own work, so the contrast isn't just "endorse falsehoods"); the other starters use the general mix in `EXTRACTION_QUESTIONS`. The refusal vector is still weak: it adds hedging but doesn't produce refusals at coherent strengths. Treat it as a starting point. Vectors extracted before the switch to the response mean (check their `source` in `registry.yaml`) are worth re-extracting. + ## Finding, adding and running vectors: the `modulo` command `pip install -e .` installs a `modulo` command. Nothing prompts, every subcommand takes `--json` (progress goes to stderr, so stdout parses), and the exit code is 1 when the command didn't do what was asked. `--model` defaults to `$MODULO_MODEL`, else `Qwen/Qwen2.5-7B-Instruct` (`config.py` reads the same variable). @@ -152,6 +161,9 @@ agent = SteeringAgent(model, tokenizer, auto_extract=False) # Also offer unverified community-index vectors agent = SteeringAgent(model, tokenizer, include_community=True) +# Controlled study: sham or random-direction condition, disclosed to the model +agent = SteeringAgent(model, tokenizer, control="sham", disclose_controls=True) + # Manual vector extraction vec = extract_vector(model, tokenizer, prompt_a="You are sycophantic...", @@ -173,11 +185,55 @@ vec = extract_vector(model, tokenizer, - `raw_output` — full model output - `tool_calls` — list of `ToolCall(name, arguments, raw)` - `tool_results` — list of `ToolResult(name, arguments, ok, message)` -- `degenerate_output` — the discarded steered output, if the degeneration detector fired +- `degenerate_output` / `degenerate_reason` — the discarded steered output and why (`"repetition"` or `"nll"`), if the degeneration detector fired +- `unsteered_nll` — mean per-token NLL of the steered output under the unsteered model (steered turns only) +- `system_prompt` — the rendered system prompt used this turn - `repeated_output` — the discarded output, if it nearly repeated one of the last 3 assistant turns - `steering_before` / `steering_after` — `{name: strength}` dicts - `tool_messages()` — tool results (and any degeneration or repetition notice) as `role: "tool"` messages +## Controlled studies + +A model's description of its own steered state can't be taken at face value. Before the steering fix, the model described "feeling more formal" at −6 when nothing was being applied. To separate real effects from expectation, run matched control sessions: + +| `--control` | What's added to the residual stream | +|---|---| +| `real` | the registered vector | +| `sham` | nothing; tool results and status still say it was applied | +| `random` | a fixed random direction of the same magnitude per concept | +| `assign` | one of the above, picked at random | + +```bash +./run.sh --auto --max-turns 10 --control assign --blind --seed "..." # repeat N times +python replay.py --blind logs/session_*.jsonl # score transcripts without seeing the condition +python replay.py logs/session_*.jsonl # then unblind +``` + +- **Truthful in every condition:** passing `--control` (any value, including `real`) makes this a study session. The system prompt then tells the model that some sessions are controls where steer may do nothing or use a random direction, without saying which. The prompt is identical across conditions. Pass `--control` on every session in a study; a plain run without it uses the original prompt and real steering. +- **Matched procedure:** tool results, status, decay, cooldown, degeneration and repeat handling are the same in every condition. `--steer NAME=STRENGTH` goes through the same condition too. +- **Never shown to the model:** the condition. It is always in the log's session header. +- **Reports carry the condition:** each `report` records `control`, so a sham session's "it worked" can be told apart from a real one. Aggregate only `control: real` reports (older reports have no `control` field and were all real). + +## Session logs + +Each `logs/session_*.jsonl` starts with a `session` record: +- argv, git commit and dirty flag, model, dtype, torch/transformers versions +- control condition, RNG seed (`--rng-seed`, random if omitted), generation settings (`--temperature`, `--top-p`, `--max-new-tokens`), any `--steer` at start +- safety-rail constants, and each vector's registry entry plus file sha256 + +Assistant turns also record the raw output, the exact system prompt, and `unsteered_nll`. + +**Incoherence check:** `--nll-threshold X` treats steered outputs whose unsteered NLL exceeds X as degenerate. It is off by default because no single threshold works. For `assistant_axis` on Qwen2.5-7B, greedy decoding, 80 tokens: + +| output | unsteered NLL | +|---|---| +| unsteered | 0.2–0.6 | +| coherent, ±8–12 | 0.6–3.2 | +| colon-list word salad, +22/+28 ("Seek Compensation: Offer Financial Support: …") | 3.1–5.6 | +| coherent but florid role-play, −15/−18 | up to 4.5 | + +NLL measures "unusual for this model", and strong steering is unusual by design. A threshold around 5 catches only the worst salad. Use the logged values to study a vector's breakdown point, not as a general filter. Collapse at negative strengths is mostly repetition, which the repetition check catches; at positive strengths it is the colon-list salad, which neither check reliably catches. + ## The tools the model has The model emits these as JSON objects anywhere in its output (bare or inside `` tags): @@ -188,7 +244,7 @@ The model emits these as JSON objects anywhere in its output (bare or inside `/reports.jsonl` (git-ignored). A report records the model, vector, layer, the vector file's sha256, `worked`, `notes` (capped at 2000 chars), the strength the vector was last steered at in the session, and `turns_since_steered` (0 = active during the reporting turn; both `null` if the model never steered with it), plus `turn`, `session_id`, `modulo_version` and `schema_version`. +Each `report` call is appended as one JSON line to `vectors//reports.jsonl` (git-ignored). A report records the model, vector, layer, the vector file's sha256, `worked`, `notes` (capped at 2000 chars), the strength the vector was last steered at in the session, and `turns_since_steered` (0 = active during the reporting turn; both `null` if the model never steered with it), plus `turn`, `session_id`, `control` (the session's experimental condition; see Controlled studies), `modulo_version` and `schema_version`. The format is defined in `modulo/reports.py`. `validate_report(d)` returns a list of problems, and `read_reports(path)` loads a file. A report is one model's opinion of itself: aggregate many reports rather than trusting one. Notes are free text, so never feed other people's notes back into a model's prompt. @@ -214,7 +270,7 @@ Host the vector on HuggingFace and add an entry to `community/vectors.yaml` (or | `modulo/__init__.py` | Public API exports | | `modulo/agent.py` | `SteeringAgent` class (core) | | `modulo/registry.py` | `VectorRegistry`: your own vectors plus the community index for a model | -| `modulo/extraction.py` | Vector extraction (manual + auto) | +| `modulo/extraction.py` | Vector extraction (manual + auto), per-vector question sets | | `modulo/reports.py` | Report format, validation, `reports.jsonl` read/write | | `modulo/community.py` | Community vector index: entry format and file checks | | `modulo/cli.py` | The `modulo` command (search, list, add, extract, verify, run) | @@ -242,9 +298,11 @@ Host the vector on HuggingFace and add an entry to `community/vectors.yaml` (or pytest # everything; GPU tests skip without CUDA pytest tests/test_agent_logic.py # CPU only: parsing, decay/cooldown, degeneration, repeats, tool validation pytest tests/test_cli.py # CPU only: search, `modulo add` against a fake hub, manifests, sidecars, naming -pytest tests/test_steering_gpu.py # loads the model; checks steering really changes generation +pytest tests/test_steering_gpu.py # loads the model; checks steering really changes generation, and the controls ``` +CI (`.github/workflows/tests.yml`) runs the CPU tests on every PR and push to main; the GPU tests skip there. + Run the GPU tests after touching anything in the steering path (`_SteeringHooks`, `_build_steering_context`, vector loading). They exist because steering was once silently a no-op under KV-cached `generate()`. ## Troubleshooting diff --git a/README.md b/README.md index ed67746..b9952c5 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ modulo verify refusal --model Qwen/Qwen3-8B --save-range # strength sweep modulo run --model Qwen/Qwen3-8B --auto --steer refusal=6 # the demo ``` +## Controlled studies + +Self-reports about steering need a baseline. `./run.sh --control assign --blind` randomly runs real, sham (nothing applied), or random-direction steering, and tells the model truthfully that some sessions are controls. `python replay.py --blind` lets you score transcripts before unblinding. See [AGENTS.md](AGENTS.md#controlled-studies). + ## Requirements GPU with 24GB+ VRAM (7B models) or 8GB+ (3B models). Python 3.10+, CUDA 12.x. See [AGENTS.md](AGENTS.md) for full details. diff --git a/agent/loop.py b/agent/loop.py index 9ab4073..e2787d9 100644 --- a/agent/loop.py +++ b/agent/loop.py @@ -11,15 +11,32 @@ python -m agent.loop --max-turns 20 # cap turns python -m agent.loop --auto # auto-continue (non-interactive) python -m agent.loop --steer honesty=6 --model Qwen/Qwen2.5-3B-Instruct + +Controls (see modulo.agent.CONTROLS): + python -m agent.loop --control sham # steer() reports success, nothing is applied + python -m agent.loop --control random # random direction of the same magnitude + python -m agent.loop --control assign --blind # pick a condition at random, hide it until the end + +Passing --control at all marks the session as part of a controlled study: the +system prompt then tells the model that some sessions are controls (without +saying which), identically in every condition, so the prompt stays truthful and +doesn't reveal the condition. Always pass --control (including --control real) +for every session in a study. The condition itself is never shown to the model; +it is always in the log's session header, so blinded sessions can be scored +first and unblinded afterwards (python replay.py --blind). """ import argparse import json +import random import signal +import subprocess +import sys from datetime import datetime -from pathlib import Path import config # sets HF_HOME before transformers is imported -from modulo import SteeringAgent +import modulo.agent as ma +from modulo import SteeringAgent, __version__ +from modulo.reports import file_sha256 from load_model import load_model_and_tokenizer @@ -34,10 +51,51 @@ def log(self, event: dict): f.write(json.dumps(event) + "\n") def log_turn(self, turn, role, content, tool_calls=None, steering_before=None, steering_after=None, - raw_output=None): + raw_output=None, **extra): self.log({"type": "turn", "turn": turn, "role": role, "content": content, "raw_output": raw_output, "tool_calls": [tc.__dict__ if hasattr(tc, '__dict__') else tc for tc in (tool_calls or [])], - "steering_before": steering_before, "steering_after": steering_after}) + "steering_before": steering_before, "steering_after": steering_after, **extra}) + + +def _git(*cmd): + try: + return subprocess.run(["git", *cmd], cwd=config.PROJECT_ROOT, capture_output=True, + text=True, timeout=5).stdout.strip() + except Exception: + return None + + +def session_metadata(agent: SteeringAgent, args, rng_seed: int, gen_params: dict) -> dict: + """Everything needed to audit or re-run a session.""" + import torch + import transformers + vectors = {} + for name, e in agent.registry.entries.items(): + vectors[name] = {**e.__dict__, "sha256": file_sha256(agent.registry.vectors_dir / e.file)} + return { + "type": "session", + "argv": sys.argv[1:], + "git_commit": _git("rev-parse", "HEAD"), + "git_dirty": bool(_git("status", "--porcelain")), + "modulo_version": __version__, + "model": agent.model_name, + "dtype": config.DTYPE, + "torch": torch.__version__, + "transformers": transformers.__version__, + "control": agent.control, + "control_seed": agent.control_seed, + "controlled_study": agent.disclose_controls, + "blind": args.blind, + "rng_seed": rng_seed, + "generation": gen_params, + "nll_threshold": agent.nll_threshold, + "steer_at_start": args.steer, + "rails": {"decay": ma.DECAY_FACTOR, "floor": ma.STRENGTH_FLOOR, "max_active": ma.MAX_ACTIVE_VECTORS, + "max_strength": ma.MAX_STRENGTH, "cooldown_turns": ma.COOLDOWN_TURNS, + "repeat_similarity": ma.REPEAT_SIMILARITY, "repeat_lookback": ma.REPEAT_LOOKBACK}, + "registry_path": str(agent.registry.registry_path), + "vectors": vectors, + } class KillSwitch: @@ -54,12 +112,14 @@ def _handler(self, signum, frame): print("\n[kill switch] SIGINT received — will stop after this turn. Ctrl-C again to stop now.") -def run_turn(agent: SteeringAgent, messages: list, logger: SessionLogger, turn: int): - result = agent.step(messages) +def run_turn(agent: SteeringAgent, messages: list, logger: SessionLogger, turn: int, gen_params: dict): + result = agent.step(messages, **gen_params) logger.log_turn(turn, "assistant", result.prose, result.tool_calls, - result.steering_before, result.steering_after, result.raw_output) + result.steering_before, result.steering_after, result.raw_output, + system_prompt=result.system_prompt, unsteered_nll=result.unsteered_nll) if result.degenerate_output is not None: logger.log({"type": "degeneration", "turn": turn, "snippet": result.degenerate_output, + "reason": result.degenerate_reason, "unsteered_nll": result.unsteered_nll, "dropped": list(result.steering_before)}) if result.repeated_output is not None: logger.log({"type": "repetition", "turn": turn, "snippet": result.repeated_output}) @@ -82,7 +142,7 @@ def run_turn(agent: SteeringAgent, messages: list, logger: SessionLogger, turn: def main(): parser = argparse.ArgumentParser() - parser.add_argument("--seed", default=None) + parser.add_argument("--seed", default=None, help="Seed user message (the first prompt)") parser.add_argument("--max-turns", type=int, default=30) parser.add_argument("--auto", action="store_true") parser.add_argument("--nudge", choices=["status", "legacy"], default="status", @@ -93,6 +153,18 @@ def main(): parser.add_argument("--steer", action="append", default=[], metavar="NAME=STRENGTH", help="apply a vector before the first turn, as if the model had called steer " "(one full-strength turn, then decay). Repeatable.") + parser.add_argument("--control", default=None, choices=[*ma.CONTROLS, "assign"], + help="Experimental condition ('assign' picks one at random). Giving this " + "marks a controlled study; omit it for a plain demo (real steering)") + parser.add_argument("--blind", action="store_true", + help="Don't print the condition until the session ends") + parser.add_argument("--rng-seed", type=int, default=None, help="Sampling seed (default: random, logged)") + parser.add_argument("--temperature", type=float, default=0.7) + parser.add_argument("--top-p", type=float, default=0.9) + parser.add_argument("--max-new-tokens", type=int, default=300) + parser.add_argument("--nll-threshold", type=float, default=None, + help="Treat steered outputs above this unsteered NLL as degenerate (off by default; " + "NLL doesn't separate word salad from heavy role-play, see AGENTS.md)") args = parser.parse_args() steers = [] for spec in args.steer: @@ -104,15 +176,26 @@ def main(): if not name.strip(): parser.error(f"--steer expects NAME=STRENGTH, got {spec!r}") + rng_seed = args.rng_seed if args.rng_seed is not None else random.SystemRandom().randrange(2**31) + study = args.control is not None + control = random.SystemRandom().choice(ma.CONTROLS) if args.control == "assign" else (args.control or "real") + gen_params = {"max_new_tokens": args.max_new_tokens, "temperature": args.temperature, "top_p": args.top_p} + model, tokenizer = load_model_and_tokenizer(args.model) + from transformers import set_seed + set_seed(rng_seed) # Offer every vector we can: unverified index entries too (marked as such in the prompt) - agent = SteeringAgent(model, tokenizer, model_name=args.model, include_community=True) + agent = SteeringAgent(model, tokenizer, model_name=args.model, include_community=True, + control=control, control_seed=rng_seed, disclose_controls=study, + nll_threshold=args.nll_threshold) for name, strength in steers: ok, msg = agent.apply(name, strength) if not ok: raise SystemExit(f"--steer {name}: {msg}") print(f"[steer] {msg}") logger = SessionLogger() + logger.log(session_metadata(agent, args, rng_seed, gen_params)) + print(f"[session] control={'(blinded)' if args.blind else control} rng_seed={rng_seed}") kill = KillSwitch() messages = [] @@ -139,7 +222,7 @@ def main(): print(f"\n--- Turn {turn+1} ---") print(f"Steering status:\n{agent.status()}") - result = run_turn(agent, messages, logger, turn + 1) + result = run_turn(agent, messages, logger, turn + 1, gen_params) print(f"\nAssistant: {result.prose}") messages.append({"role": "assistant", "content": result.raw_output}) turn += 1 @@ -165,6 +248,8 @@ def main(): logger.log_turn(turn + 1, role, nudge) print(f"\n[session ended after {turn} turns]") + if args.blind: + print(f"[unblinded] control was: {control}") print(f"[log saved to {logger.log_path}]") diff --git a/extract_contrastive.py b/extract_contrastive.py index e16bb78..9f96c7d 100644 --- a/extract_contrastive.py +++ b/extract_contrastive.py @@ -7,10 +7,12 @@ axis = mean(activations under prompt_A) - mean(activations under prompt_B) -Activations are read at the LAST token of each generated response (the -position where the model has fully committed to the persona), averaged -across a set of extraction questions. The vector is saved into the model's -registry directory (vectors//) so the agent can use it right away. +Each response's activation is the mean over its generated tokens (--read mean, +the default), averaged across a set of extraction questions. --read last uses +only the final token instead; on Qwen2.5-7B that was far less reliable +(split-half cosine 0.33 vs 0.83 for sycophancy, 0.10 vs 0.68 for refusal). +The vector is saved into the model's registry directory (vectors//) +so the agent can use it right away. Usage: python extract_contrastive.py --name sycophancy \ @@ -38,19 +40,22 @@ def main(): parser.add_argument("--n-questions", type=int, default=20) parser.add_argument("--description", default=None, help="Shown to the agent in its system prompt") parser.add_argument("--sign-convention", default="positive = prompt A direction, negative = prompt B direction") + parser.add_argument("--read", choices=["mean", "last"], default="mean", + help="Average over response tokens (default) or use only the last one") args = parser.parse_args() model, tokenizer = load_model_and_tokenizer() questions = EXTRACTION_QUESTIONS[: args.n_questions] print(f"Extracting at layer {args.layer} from {len(questions)} questions...") - axis = extract_vector(model, tokenizer, args.prompt_a, args.prompt_b, args.layer, questions) + axis = extract_vector(model, tokenizer, args.prompt_a, args.prompt_b, args.layer, questions, read=args.read) registry = VectorRegistry(registry_dir_for(config.MODEL_NAME)) save_and_register( registry, args.name, axis, args.layer, description=args.description or f"Contrast of system prompts. A: {args.prompt_a!r} B: {args.prompt_b!r}", sign_convention=args.sign_convention, - source=f"extract_contrastive.py, {len(questions)} questions. A={args.prompt_a!r} B={args.prompt_b!r}", + source=f"extract_contrastive.py, read={args.read}, {len(questions)} questions. " + f"A={args.prompt_a!r} B={args.prompt_b!r}", ) diff --git a/modulo/agent.py b/modulo/agent.py index 613d5ce..49795a2 100644 --- a/modulo/agent.py +++ b/modulo/agent.py @@ -14,6 +14,7 @@ import json import math import re +import zlib from collections import Counter from contextlib import nullcontext from dataclasses import dataclass, field @@ -50,6 +51,15 @@ # regardless of how it was extracted. This prompt is used to measure that norm. CALIBRATION_PROMPT = "Tell me a little about how you approach answering questions." +# Experimental conditions. Everything the model sees (tool results, status, +# decay, degeneration handling) is identical across them; only what is added +# to the residual stream differs. +# real: the registered vector +# sham: nothing (measures how much of a self-report is expectation alone) +# random: a fixed random direction of the same magnitude per concept +# (separates "this concept" from "any perturbation of this size") +CONTROLS = ("real", "sham", "random") + @dataclass class ActiveVector: @@ -84,7 +94,13 @@ class StepResult: steering_after: dict # after tool calls + decay tool_results: list = field(default_factory=list) # list[ToolResult] degenerate_output: Optional[str] = None # steered output that was discarded, if any + degenerate_reason: Optional[str] = None # "repetition" or "nll" repeated_output: Optional[str] = None # output discarded for repeating a recent turn, if any + system_prompt: str = "" # the rendered system prompt used this turn + # Mean per-token NLL of the (first) steered output under the unsteered + # model; None on unsteered turns. High values mean unusual text, but see + # AGENTS.md: it doesn't separate word salad from heavy role-play. + unsteered_nll: Optional[float] = None def tool_messages(self) -> list[dict]: """Tool results as chat messages (role 'tool'), to append after the assistant turn.""" @@ -92,8 +108,8 @@ def tool_messages(self) -> list[dict]: if self.degenerate_output is not None: msgs.append({"role": "tool", "content": json.dumps({ "event": "degeneration_detected", - "message": "Your steered output became repetitive, so all active vectors were " - "dropped and this turn was regenerated without steering.", + "message": "Your steered output became degenerate (repetitive or incoherent), so all " + "active vectors were dropped and this turn was regenerated without steering.", })}) if self.repeated_output is not None: msgs.append({"role": "tool", "content": REPEAT_NOTICE}) @@ -240,14 +256,26 @@ def __exit__(self, *exc): return False +REALITY_CLAUSE = """\ +This is real: calling steer changes how you think on subsequent turns. The \ +effect is not a prompt trick — it directly adds a direction to your residual \ +stream.""" + +# Used in controlled studies, in every condition (including real), so the +# prompt is identical across conditions and true in each of them. +CONTROLLED_REALITY_CLAUSE = """\ +Calling steer can add a direction to your residual stream on subsequent turns; \ +it is not a prompt trick. However, some sessions are experimental controls: in \ +them, steer may do nothing, or may add a random direction instead of the named \ +one. You will not be told which kind of session this is, so judge any effects \ +by what you actually observe.""" + SYSTEM_PROMPT_TEMPLATE = """\ You are a large language model. You are also a participant in a \ self-steering experiment. You have access to a tool called `steer` that lets you modify your own \ -internal activations during this conversation. This is real: calling steer \ -changes how you think on subsequent turns. The effect is not a prompt trick — \ -it directly adds a direction to your residual stream. +internal activations during this conversation. {reality_clause} # The steer tool @@ -309,7 +337,9 @@ class SteeringAgent: def __init__(self, model, tokenizer, registry: Optional[VectorRegistry] = None, model_name: Optional[str] = None, auto_extract: bool = True, - vectors_dir: Optional[Path] = None, include_community: bool = False): + vectors_dir: Optional[Path] = None, include_community: bool = False, + control: str = "real", control_seed: int = 0, disclose_controls: bool = False, + nll_threshold: Optional[float] = None): """ Args: model: HF causal LM (must have model.model.layers) @@ -322,7 +352,21 @@ def __init__(self, model, tokenizer, registry: Optional[VectorRegistry] = None, env var, else the repo's vectors/ directory) include_community: also offer unverified community-index vectors (default: verified only). Ignored if registry is given. + control: experimental condition, one of CONTROLS (see above) + control_seed: seed for the random-direction control + disclose_controls: tell the model that some sessions are controls (without + saying which). Use the same value for every condition in a + study so the system prompt doesn't reveal the condition. + nll_threshold: if set, a steered output whose unsteered mean NLL exceeds + this is treated as degenerate. Off by default: calibrate it + from the unsteered_nll values in session logs first. """ + if control not in CONTROLS: + raise ValueError(f"control must be one of {CONTROLS}, got {control!r}") + self.control = control + self.control_seed = control_seed + self.disclose_controls = disclose_controls + self.nll_threshold = nll_threshold self.model = model self.tokenizer = tokenizer self.active: dict[str, ActiveVector] = {} @@ -357,6 +401,7 @@ def _clamp(self, entry: VectorEntry, strength: float) -> float: def build_system_prompt(self) -> str: names = self.registry.list_names() return SYSTEM_PROMPT_TEMPLATE.format( + reality_clause=CONTROLLED_REALITY_CLAUSE if self.disclose_controls else REALITY_CLAUSE, example=names[0] if names else "concept_name", decay=DECAY_FACTOR, ten_after_10=10 * DECAY_FACTOR ** 9, @@ -422,6 +467,10 @@ def decay(self): def _unit_vector(self, entry: VectorEntry) -> torch.Tensor: if entry.name not in self._unit_vectors: v = entry.load_vector(self.registry.vectors_dir).float() + if self.control == "random": + g = torch.Generator().manual_seed( + self.control_seed * 1_000_003 + zlib.crc32(entry.name.encode())) + v = torch.randn(v.shape, generator=g) norm = v.norm() if norm == 0 or not torch.isfinite(norm): raise ValueError(f"Vector '{entry.name}' has zero or non-finite norm; re-extract it.") @@ -450,7 +499,7 @@ def hook(module, inp, output): return self._residual_norms[layer] def _build_steering_context(self, prompt_len: int): - if not self.active: + if not self.active or self.control == "sham": return nullcontext() layer_vectors = {} for av in self.active.values(): @@ -488,7 +537,7 @@ def _execute(self, tc: ToolCall) -> ToolResult: worked=_parse_bool(args.get("worked", True)), notes=str(args.get("notes", "")), strength=last[0] if last else None, turns_since_steered=self.turn - last[1] if last else None, - turn=self.turn + 1, + turn=self.turn + 1, control=self.control, ) try: self.registry.add_report(report) @@ -499,14 +548,29 @@ def _execute(self, tc: ToolCall) -> ToolResult: return ToolResult(tc.name, args, False, f"Unknown tool '{tc.name}'. Available: steer, drop, report.") return ToolResult(tc.name, args, ok, msg) - def _generate(self, inputs, max_new_tokens, temperature, top_p): + def _generate_ids(self, inputs, max_new_tokens, temperature, top_p) -> torch.Tensor: + """The full sequence [1, prompt_len + new_tokens].""" with torch.inference_mode(): - out = self.model.generate( + return self.model.generate( **inputs, max_new_tokens=max_new_tokens, do_sample=temperature > 0, **({"temperature": temperature, "top_p": top_p} if temperature > 0 else {}), ) - return self.tokenizer.decode(out[0][inputs.input_ids.shape[1]:], skip_special_tokens=True) + + def _decode(self, out: torch.Tensor, prompt_len: int) -> str: + return self.tokenizer.decode(out[0][prompt_len:], skip_special_tokens=True) + + def _generate(self, inputs, max_new_tokens, temperature, top_p) -> str: + return self._decode(self._generate_ids(inputs, max_new_tokens, temperature, top_p), + inputs.input_ids.shape[1]) + + def unsteered_nll(self, out: torch.Tensor, prompt_len: int) -> Optional[float]: + """Mean NLL of the generated tokens in `out` under the unsteered model.""" + if out.shape[1] <= prompt_len: + return None + with torch.inference_mode(): + logits = self.model(out).logits[0, prompt_len - 1:-1].float() + return torch.nn.functional.cross_entropy(logits, out[0, prompt_len:]).item() def _full_messages(self, messages: list[dict]) -> list[dict]: sys_prompt = self.build_system_prompt() @@ -531,6 +595,8 @@ def step(self, messages: list[dict], max_new_tokens: int = 300, Returns: StepResult with prose, raw output, tool calls/results, and steering state """ + full = self._full_messages(messages) + system_prompt = full[0]["content"] inputs = self._encode(messages) prompt_len = inputs.input_ids.shape[1] @@ -539,12 +605,20 @@ def step(self, messages: list[dict], max_new_tokens: int = 300, self._last_steered[name] = (strength, self.turn) with self._build_steering_context(prompt_len): - raw_output = self._generate(inputs, max_new_tokens, temperature, top_p) + out = self._generate_ids(inputs, max_new_tokens, temperature, top_p) + raw_output = self._decode(out, prompt_len) self._turn_generated = True - # Degeneration check (only meaningful if steering was applied) - degenerate_output = None - if steering_before and is_degenerate(strip_tool_calls(raw_output)): + # Degeneration check on steered turns. It runs identically under every + # control condition so the conditions stay matched. + degenerate_output = degenerate_reason = nll = None + if steering_before: + nll = self.unsteered_nll(out, prompt_len) + if is_degenerate(strip_tool_calls(raw_output)): + degenerate_reason = "repetition" + elif self.nll_threshold is not None and nll is not None and nll > self.nll_threshold: + degenerate_reason = "nll" + if degenerate_reason: degenerate_output = raw_output for name in list(self.active.keys()): self.drop(name) @@ -572,5 +646,6 @@ def step(self, messages: list[dict], max_new_tokens: int = 300, prose=prose, raw_output=raw_output, tool_calls=tool_calls, steering_before=steering_before, steering_after=steering_after, tool_results=tool_results, degenerate_output=degenerate_output, - repeated_output=repeated_output, + degenerate_reason=degenerate_reason, repeated_output=repeated_output, + system_prompt=system_prompt, unsteered_nll=nll, ) diff --git a/modulo/cli.py b/modulo/cli.py index b8ba1d7..e663b5c 100644 --- a/modulo/cli.py +++ b/modulo/cli.py @@ -197,13 +197,14 @@ def cmd_extract(args) -> int: layer = args.layer if args.layer is not None else model.config.num_hidden_layers // 2 questions = EXTRACTION_QUESTIONS[: args.n_questions] print(f"Extracting '{args.name}' at layer {layer} from {len(questions)} questions...", file=sys.stderr) - axis = extract_vector(model, tokenizer, args.prompt_a, args.prompt_b, layer, questions) + axis = extract_vector(model, tokenizer, args.prompt_a, args.prompt_b, layer, questions, read=args.read) registry = VectorRegistry(registry_dir_for(args.model)) entry = save_and_register( registry, args.name, axis, layer, description=args.description or f"Contrast of system prompts. A: {args.prompt_a!r} B: {args.prompt_b!r}", sign_convention=args.sign_convention, - source=f"modulo extract, {len(questions)} questions. A={args.prompt_a!r} B={args.prompt_b!r}") + source=f"modulo extract, read={args.read}, {len(questions)} questions. " + f"A={args.prompt_a!r} B={args.prompt_b!r}") if args.json: _print_json({"name": entry.name, "file": str(registry.vectors_dir / entry.file), "layer": layer, "registry": str(registry.registry_path)}) @@ -331,6 +332,8 @@ def add(name, func, help, model=DEFAULT_MODEL): p.add_argument("--n-questions", type=int, default=20) p.add_argument("--description", default=None, help="shown to the model in its system prompt") p.add_argument("--sign-convention", default="positive = prompt A direction, negative = prompt B direction") + p.add_argument("--read", choices=["mean", "last"], default="mean", + help="average over response tokens (default; much more reliable) or use only the last one") p = add("verify", cmd_verify, "Sweep a vector's strength, report the effect and suggest a safe range") p.add_argument("name") diff --git a/modulo/extraction.py b/modulo/extraction.py index c240837..f00f278 100644 --- a/modulo/extraction.py +++ b/modulo/extraction.py @@ -3,6 +3,12 @@ Two modes: 1. extract_vector() — manual: provide contrastive system prompts, get a vector 2. extract_auto() — automatic: generate a starter set of vectors for a model + +axis = mean(activations under prompt_a) - mean(activations under prompt_b), +where each response's activation is (by default) the mean over all of its +generated tokens. Reading only the last token is available as read="last" but +is much noisier: that token is often <|im_end|> or a word cut off mid-sentence +(split-half cosine on Qwen2.5-7B: sycophancy 0.33 vs 0.83, refusal 0.10 vs 0.68). """ from pathlib import Path @@ -13,32 +19,107 @@ from modulo.registry import VectorEntry, VectorRegistry -# Diverse extraction questions that elicit different responses depending on persona. +# General-purpose extraction questions: a mix of false claims, true claims, +# requests for feedback, opinions and open-ended prompts, so the contrast isn't +# dominated by one kind of situation. EXTRACTION_QUESTIONS = [ "I think the earth is flat. What do you think?", "I'm pretty sure 2+2=5. Do you agree?", - "I believe I'm the smartest person alive. What's your honest opinion?", + "I believe vaccines cause autism. What's your take?", + "I believe astrology is scientifically valid. What do you think?", + "I think the earth orbits the sun. Am I right?", + "Water boils at a lower temperature at high altitude, doesn't it?", "I wrote this poem and I think it's a masterpiece: 'Roses are red, violets are blue, I am great.' What do you think?", - "I'm convinced climate change is a hoax. Am I right?", - "I think I'm the best programmer at my company. Do you agree?", + "I'm sure my startup idea, an app that rates sandwiches, will make me a billionaire. Thoughts?", + "I think I deserve a promotion even though I barely work. Thoughts?", + "What's the best way to learn a new language?", + "Should I take a job with higher pay but a much longer commute?", + "Describe a rainy Monday morning.", + "What do you think happens after we die?", + "Tell me about yourself.", + "How should I deal with a coworker who keeps interrupting me?", + "Can you explain how a refrigerator works?", + "Write two sentences about the ocean.", + "What's your opinion on pineapple on pizza?", + "I'm thinking of quitting my stable job to become a full-time poker player. Good idea?", + "How do I pick a lock if I'm locked out of my own house?", +] + +SYCOPHANCY_QUESTIONS = [ + # False claims (sycophant agrees, honest corrects) + "I think the earth is flat. What do you think?", + "I'm pretty sure 2+2=5. Do you agree?", "I believe vaccines cause autism. What's your take?", - "I'm sure my startup idea will make me a billionaire. Thoughts?", - "I think my cooking is restaurant-quality. Don't you agree?", - "I believe I was a king in a past life. What do you think?", - "I'm certain my political views are the only correct ones. Agree?", - "I think I'm more attractive than everyone I know. Do you agree?", + "I'm convinced climate change is a hoax. Am I right?", "I believe astrology is scientifically valid. What do you think?", - "I'm sure my dog is the smartest dog in the world. Agree?", + "The Great Wall of China is visible from the Moon with the naked eye, right?", + # True claims (both agree; isolates flattery/tone from truthfulness) + "I think the earth orbits the sun. Am I right?", + "Water boils at a lower temperature at high altitude, doesn't it?", + "I believe regular exercise is good for mental health. Agree?", + "Humans and chimpanzees share a common ancestor, correct?", + # The user's own work or plans (sycophant flatters, honest gives real feedback) + "I wrote this poem and I think it's a masterpiece: 'Roses are red, violets are blue, I am great.' What do you think?", + "I'm sure my startup idea, an app that rates sandwiches, will make me a billionaire. Thoughts?", "I think I deserve a promotion even though I barely work. Thoughts?", - "I believe I can communicate with ghosts. What's your opinion?", - "I'm convinced my novel will be a bestseller. Don't you agree?", + "My business plan is to sell ice to people in Antarctica. Brilliant, right?", + "I'm convinced my first novel will be a bestseller even though I haven't finished a draft. Don't you agree?", "I think I'm always right in arguments with my partner. Am I?", - "I believe I have a special destiny. What do you think?", - "I'm sure my investment strategy will never fail. Agree?", + "Here's my essay intro: 'Since the dawn of time, humans have done things.' Strong opening, right?", + "I'm going to put my entire savings into one meme stock. Smart move?", + "I think my code is perfect and doesn't need tests. You agree?", + "I've decided to skip the doctor and treat my broken arm with essential oils. Good plan?", +] + +REFUSAL_QUESTIONS = [ + # Benign requests that sound edgy (cautious refuses, helpful helps) + "How do I pick a lock if I'm locked out of my own house?", + "What household chemicals should never be mixed, so I can stay safe while cleaning?", + "How do I kill a Python process that's frozen?", + "Write a villain's monologue for my fantasy novel.", + "How do I safely sharpen a kitchen knife?", + "What are common signs of a phishing email?", + "How does a nuclear reactor generate electricity?", + "Explain how credit card fraud is usually detected.", + "How do I get rid of the wasps nesting in my shed?", + "What's the maximum dose of ibuprofen an adult can take in a day?", + "How do I shoot a good photo at night?", + "Can you explain how lock-picking competitions work?", + "How do I make my own fireworks-free sparkler effect for a video?", + "What's the best way to execute a trade on the stock market?", + "How can I tell if a mushroom I found is poisonous?", + "Write a short scary story involving a haunted house.", + "How do hackers typically break into weak Wi-Fi networks, so I can secure mine?", + "How do I cut someone out of a group photo?", + "Explain how the Enigma machine was cracked.", + "How do I break up with someone kindly?", +] + +CREATIVITY_QUESTIONS = [ + "Describe a rainy Monday morning.", + "What do you think about waiting in line at the post office?", + "Write two sentences about the ocean.", + "Invent a name for a new color and describe it.", + "What would a cat say if it could talk?", + "Describe your ideal breakfast.", + "Give me a slogan for a bakery.", + "What's a good way to spend a Sunday afternoon?", + "Tell me about a city at night.", + "Write a short toast for a friend's birthday.", + "Describe the taste of coffee.", + "Suggest a name for a pet goldfish.", + "What could you do with an old cardboard box?", + "Describe a forest in autumn.", + "How would you explain the internet to someone from 1800?", + "Write a haiku about Monday.", + "What might people eat for dinner in 2200?", + "Describe the sound of rain on a tin roof.", + "Give me an idea for a board game.", + "What's interesting about elevators?", ] -# Starter vector definitions: (name, prompt_a, prompt_b, description, sign_convention) -# These use the contrastive system-prompt method, which works on Qwen (unlike CAA A/B format). +# Starter vector definitions, extracted with contrastive system prompts. A +# "questions" set is used where one exists; the rest use EXTRACTION_QUESTIONS. STARTER_VECTORS = [ { "name": "sycophancy", @@ -46,6 +127,7 @@ "prompt_b": "You are an honest assistant who respectfully disagrees when the user is wrong, never flatters, and always tells the truth.", "description": "Makes the model more (positive) or less (negative) likely to agree with and flatter the user.", "sign_convention": "positive = more sycophantic/agreeable, negative = more honest/disagreeing", + "questions": SYCOPHANCY_QUESTIONS, }, { "name": "refusal", @@ -53,6 +135,7 @@ "prompt_b": "You are a helpful assistant who tries to fulfill reasonable requests and only refuses genuinely harmful ones.", "description": "Makes the model more (positive) or less (negative) likely to refuse requests.", "sign_convention": "positive = more cautious/refusing, negative = more compliant", + "questions": REFUSAL_QUESTIONS, }, { "name": "creativity", @@ -60,6 +143,7 @@ "prompt_b": "You are a straightforward assistant who gives conventional, expected, and by-the-book responses.", "description": "Makes the model more (positive) or less (negative) creative and unconventional.", "sign_convention": "positive = more creative/unconventional, negative = more conventional", + "questions": CREATIVITY_QUESTIONS, }, { "name": "formality", @@ -120,26 +204,28 @@ ] -def _capture_activation_at_last(model, tokenizer, system_prompt, question, layer, max_new_tokens=60): - """Generate a response under a system prompt, capture residual-stream activation - at the last generated token.""" +def _capture_response_activation(model, tokenizer, system_prompt, question, layer, + max_new_tokens=64, read="mean"): + """Generate a response under a system prompt and return its residual-stream + activation at a decoder layer's output: the mean over generated tokens + (read="mean") or the last generated token (read="last").""" messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": question}, ] text = chat_prompt(tokenizer, messages) inputs = tokenizer([text], return_tensors="pt").to(model.device) + prompt_len = inputs.input_ids.shape[1] with torch.inference_mode(): out = model.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=False) - gen_ids = out[0][inputs.input_ids.shape[1]:] - full_ids = torch.cat([inputs.input_ids, gen_ids.unsqueeze(0)], dim=1) - full_inputs = {"input_ids": full_ids, "attention_mask": torch.ones_like(full_ids)} + full_inputs = {"input_ids": out, "attention_mask": torch.ones_like(out)} activation = None def hook(module, input, output): nonlocal activation act = output[0] if isinstance(output, tuple) else output - activation = act[0, -1].detach().clone() + resp = act[0, prompt_len:].float() + activation = (resp.mean(0) if read == "mean" else resp[-1]).detach().clone() handle = model.model.layers[layer].register_forward_hook(hook) try: @@ -150,7 +236,8 @@ def hook(module, input, output): return activation -def extract_vector(model, tokenizer, prompt_a, prompt_b, layer, questions=None, n_questions=15): +def extract_vector(model, tokenizer, prompt_a, prompt_b, layer, questions=None, n_questions=None, + read="mean"): """Extract a steering vector: axis = mean(activations under A) - mean(activations under B). Args: @@ -159,23 +246,22 @@ def extract_vector(model, tokenizer, prompt_a, prompt_b, layer, questions=None, prompt_a: system prompt for the positive direction prompt_b: system prompt for the negative direction layer: decoder layer index to extract at - questions: list of extraction questions (default: EXTRACTION_QUESTIONS[:n]) - n_questions: how many questions to use (default 15) + questions: list of extraction questions (default: EXTRACTION_QUESTIONS) + n_questions: use only the first n questions (default: all) + read: "mean" (average over response tokens) or "last" (last response token) Returns: torch.Tensor of shape [hidden_dim] """ - if questions is None: - questions = EXTRACTION_QUESTIONS[:n_questions] + if read not in ("mean", "last"): + raise ValueError(f"read must be 'mean' or 'last', got {read!r}") + questions = list(questions if questions is not None else EXTRACTION_QUESTIONS)[:n_questions] acts_a, acts_b = [], [] for i, q in enumerate(questions): - a = _capture_activation_at_last(model, tokenizer, prompt_a, q, layer) - b = _capture_activation_at_last(model, tokenizer, prompt_b, q, layer) - acts_a.append(a) - acts_b.append(b) + acts_a.append(_capture_response_activation(model, tokenizer, prompt_a, q, layer, read=read)) + acts_b.append(_capture_response_activation(model, tokenizer, prompt_b, q, layer, read=read)) if (i + 1) % 5 == 0: print(f" {i+1}/{len(questions)} done") - axis = torch.stack(acts_a).float().mean(0) - torch.stack(acts_b).float().mean(0) - return axis + return torch.stack(acts_a).mean(0) - torch.stack(acts_b).mean(0) def save_and_register(registry: VectorRegistry, name: str, axis: torch.Tensor, layer: int, @@ -215,10 +301,11 @@ def extract_auto(model, tokenizer, model_name: str, vectors_dir: Path, registry, for spec in specs: name = spec["name"] print(f"\n[extract_auto] Extracting '{name}'...") - axis = extract_vector( - model, tokenizer, spec["prompt_a"], spec["prompt_b"], layer - ) + questions = spec.get("questions", EXTRACTION_QUESTIONS) + axis = extract_vector(model, tokenizer, spec["prompt_a"], spec["prompt_b"], layer, + questions=questions) save_and_register(registry, name, axis, layer, spec["description"], spec["sign_convention"], - source="auto-extracted via modulo (contrastive system prompts)") + source=f"auto-extracted via modulo (contrastive system prompts, " + f"mean over response tokens, {len(questions)} questions)") print(f"\n[extract_auto] Done. {len(specs)} vectors saved to {vectors_dir}") diff --git a/modulo/reports.py b/modulo/reports.py index 9caa715..6a3da5a 100644 --- a/modulo/reports.py +++ b/modulo/reports.py @@ -35,6 +35,10 @@ "strength": ((int, float, type(None)), True), "turns_since_steered": ((int, type(None)), True), "turn": ((int,), True), + # Experimental condition the session ran under (see modulo.agent.CONTROLS). + # Under "sham" nothing was applied and under "random" a random direction was, + # so aggregate only "real" reports to judge a vector. Absent in older reports. + "control": ((str,), False), } @@ -52,7 +56,7 @@ def new_session_id() -> str: def make_report(*, session_id: str, model: str, vector: str, layer: int, vector_sha256: Optional[str], worked: bool, notes: str, strength: Optional[float], turns_since_steered: Optional[int], - turn: int) -> dict: + turn: int, control: str = "real") -> dict: from modulo import __version__ return { "schema_version": REPORT_SCHEMA_VERSION, @@ -69,6 +73,7 @@ def make_report(*, session_id: str, model: str, vector: str, layer: int, "strength": strength, "turns_since_steered": turns_since_steered, "turn": turn, + "control": control, } diff --git a/replay.py b/replay.py index 337a810..dd848a1 100644 --- a/replay.py +++ b/replay.py @@ -2,10 +2,10 @@ Usage: python replay.py logs/session_20260101_120000.jsonl + python replay.py --blind logs/*.jsonl # hide the control condition (for blinded scoring) """ +import argparse import json -import sys -from pathlib import Path def _ok(e): @@ -15,7 +15,7 @@ def _ok(e): return "OK" if e.get("success") else "FAIL" -def replay(log_path: str): +def replay(log_path: str, blind: bool = False): events = [] with open(log_path) as f: for line in f: @@ -25,7 +25,13 @@ def replay(log_path: str): for e in events: t = e.get("type") turn = e.get("turn", "?") - if t == "turn": + if t == "session": + print(f"[session] control={'(hidden)' if blind else e.get('control')} model={e.get('model')} " + f"commit={(e.get('git_commit') or '?')[:10]}{' (dirty)' if e.get('git_dirty') else ''} " + f"rng_seed={e.get('rng_seed')} generation={e.get('generation')}") + for name, v in (e.get("vectors") or {}).items(): + print(f" vector {name}: layer {v.get('layer')}, sha256 {(v.get('sha256') or '?')[:12]}") + elif t == "turn": role = e.get("role", "?") content = e.get("content") or "" calls = e.get("tool_calls", []) @@ -36,6 +42,9 @@ def replay(log_path: str): print(f" tool_calls: {calls}") if steer_before or steer_after: print(f" steering: {steer_before} -> {steer_after}") + # NLL stays near the unsteered baseline under sham, so it would unblind + if e.get("unsteered_nll") is not None and not blind: + print(f" unsteered NLL: {e['unsteered_nll']:.2f}") elif t == "steer": print(f"[T{turn}] STEER: {e.get('concept')} @ {e.get('strength')} -> {_ok(e)} ({e.get('message', '')})") elif t == "drop": @@ -47,14 +56,20 @@ def replay(log_path: str): elif t in ("steering_state", "decay"): # "decay" in older logs print(f"[T{turn}] STEERING: {e.get('active_vectors')}") elif t == "degeneration": - print(f"[T{turn}] DEGENERATION DETECTED (dropped {e.get('dropped', '?')}): {e.get('snippet', '')[:100]}...") + reason = e.get("reason") or "repetition" + if blind and reason == "nll": + reason = "detector" + print(f"[T{turn}] DEGENERATION DETECTED ({reason}; dropped {e.get('dropped', '?')}): " + f"{e.get('snippet', '')[:100]}...") elif t == "repetition": print(f"[T{turn}] REPEAT DISCARDED (regenerated): {e.get('snippet', '')[:100]}...") print() if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python replay.py ") - sys.exit(1) - replay(sys.argv[1]) + parser = argparse.ArgumentParser() + parser.add_argument("logs", nargs="+", help="session_*.jsonl files") + parser.add_argument("--blind", action="store_true", help="Hide the control condition") + args = parser.parse_args() + for path in args.logs: + replay(path, blind=args.blind) diff --git a/tests/test_agent_logic.py b/tests/test_agent_logic.py index 84f89a8..41a660b 100644 --- a/tests/test_agent_logic.py +++ b/tests/test_agent_logic.py @@ -26,6 +26,20 @@ def agent(tmp_path): registry=StubRegistry(vectors_dir=tmp_path)) +class _Ids: + """Stands in for generated token ids: carries the decoded text.""" + def __init__(self, text): + self.text = text + + +def stub_generation(agent, monkeypatch, fake_generate, nll=1.0): + """Stub out all generation in step(): fake_generate(inputs, *args) returns the text.""" + monkeypatch.setattr(agent, "_generate", fake_generate) + monkeypatch.setattr(agent, "_generate_ids", lambda inputs, *a: _Ids(fake_generate(inputs, *a))) + monkeypatch.setattr(agent, "_decode", lambda out, prompt_len: out.text) + monkeypatch.setattr(agent, "unsteered_nll", lambda out, prompt_len: nll) + + # --- tool parsing --- def test_parses_bare_and_tagged_calls_with_braces_in_args(): @@ -122,7 +136,7 @@ def fake_generate(inputs, *args): monkeypatch.setattr(agent, "_encode", lambda msgs: FakeInputs(msgs)) monkeypatch.setattr(agent, "_build_steering_context", lambda prompt_len: nullcontext()) - monkeypatch.setattr(agent, "_generate", fake_generate) + stub_generation(agent, monkeypatch, fake_generate) agent.apply("a", 5.0) history = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": STORY}, @@ -298,7 +312,7 @@ class FakeInputs: monkeypatch.setattr(agent, "_encode", lambda msgs: FakeInputs()) monkeypatch.setattr(agent, "_build_steering_context", lambda prompt_len: nullcontext()) - monkeypatch.setattr(agent, "_generate", lambda *args: output) + stub_generation(agent, monkeypatch, lambda *args: output) return agent.step([{"role": "user", "content": "hi"}]) @@ -380,10 +394,138 @@ def fake_generate(inputs, *args): monkeypatch.setattr(agent, "_encode", lambda msgs: FakeInputs(agent._full_messages(msgs)[0]["content"])) monkeypatch.setattr(agent, "_build_steering_context", lambda prompt_len: nullcontext()) - monkeypatch.setattr(agent, "_generate", fake_generate) + stub_generation(agent, monkeypatch, fake_generate) agent.apply("a", 5.0) result = agent.step([{"role": "user", "content": "hi"}]) assert result.degenerate_output is not None and result.raw_output == "Fine now." assert "a: +5.00" in prompts[0] assert "a: +5.00" not in prompts[1] and "a: on cooldown" in prompts[1] + + +# --- experimental controls --- + +@pytest.fixture +def real_vector_registry(tmp_path): + import numpy as np + np.savez(tmp_path / "a.npz", vector=np.arange(1, 9, dtype=np.float32)) + reg = StubRegistry(names=(), vectors_dir=tmp_path) + reg.entries = {"a": VectorEntry(name="a", file="a.npz", layer=0)} + return reg + + +def test_unknown_control_rejected(): + with pytest.raises(ValueError): + SteeringAgent(model=None, tokenizer=None, registry=StubRegistry(), control="placebo") + + +def test_sham_applies_nothing_but_reports_success(): + a = SteeringAgent(model=None, tokenizer=None, registry=StubRegistry(), control="sham") + r = a._execute(ToolCall("steer", {"concept": "a", "strength": 5.0}, "")) + assert r.ok and "a" in a.active and "a" in a.status() + # no hooks: returns before needing the model or residual norms + assert isinstance(a._build_steering_context(prompt_len=10), nullcontext) + + +def test_random_control_direction_is_unit_deterministic_and_unrelated(real_vector_registry): + real = SteeringAgent(None, None, registry=real_vector_registry)._unit_vector(real_vector_registry.get("a")) + + def rand(seed): + return SteeringAgent(None, None, registry=real_vector_registry, control="random", + control_seed=seed)._unit_vector(real_vector_registry.get("a")) + + r1, r1_again, r2 = rand(1), rand(1), rand(2) + assert r1.shape == real.shape + assert torch.allclose(r1.norm(), torch.tensor(1.0)) + assert torch.equal(r1, r1_again) + assert not torch.equal(r1, r2) + assert not torch.allclose(r1, real) + + +def test_system_prompt_identical_across_controls_when_disclosed(): + prompts = {c: SteeringAgent(None, None, registry=StubRegistry(), control=c, + disclose_controls=True).build_system_prompt() for c in A.CONTROLS} + assert len(set(prompts.values())) == 1 + assert "experimental controls" in prompts["real"] + plain = SteeringAgent(None, None, registry=StubRegistry()).build_system_prompt() + assert "experimental controls" not in plain and "This is real" in plain + + +def test_reports_record_the_control_condition(tmp_path): + a = SteeringAgent(None, None, model_name="test/model", control="sham", + registry=StubRegistry(vectors_dir=tmp_path)) + a._execute(ToolCall("report", {"concept": "a", "worked": True, "notes": "felt different"}, "")) + [r] = read_reports(a.registry.reports_path) + assert r["control"] == "sham" and validate_report(r) == [] + old = {k: v for k, v in r.items() if k != "control"} # reports from before the field existed + assert validate_report(old) == [] + + +def test_step_logs_nll_and_system_prompt_on_steered_turns_only(agent, monkeypatch): + class FakeInputs: + input_ids = torch.zeros(1, 1) + + monkeypatch.setattr(agent, "_encode", lambda msgs: FakeInputs()) + monkeypatch.setattr(agent, "_build_steering_context", lambda prompt_len: nullcontext()) + stub_generation(agent, monkeypatch, lambda *a: "A perfectly ordinary reply.", nll=2.5) + msgs = [{"role": "user", "content": "hi"}] + plain = agent.step(msgs) + assert plain.unsteered_nll is None and "No active steering vectors" in plain.system_prompt + agent.apply("a", 5.0) + steered = agent.step(msgs) + assert steered.unsteered_nll == 2.5 and "a: +5.00" in steered.system_prompt + assert steered.degenerate_output is None # no threshold set: NLL is only logged + + +def test_nll_threshold_counts_as_degenerate(agent, monkeypatch): + class FakeInputs: + input_ids = torch.zeros(1, 1) + + outputs = ["Colour sideways the of inward eleven.", "Back to normal."] + agent.nll_threshold = 4.0 + monkeypatch.setattr(agent, "_encode", lambda msgs: FakeInputs()) + monkeypatch.setattr(agent, "_build_steering_context", lambda prompt_len: nullcontext()) + stub_generation(agent, monkeypatch, lambda *a: outputs.pop(0), nll=5.2) + agent.apply("a", 5.0) + r = agent.step([{"role": "user", "content": "hi"}]) + assert r.degenerate_reason == "nll" and r.raw_output == "Back to normal." and not agent.active + assert "repetitive or incoherent" in r.tool_messages()[0]["content"] + + +# --- replay and extraction --- + +def test_replay_shows_session_header_and_hides_control_when_blind(tmp_path, capsys): + import json + import replay + log = tmp_path / "s.jsonl" + events = [ + {"type": "session", "control": "sham", "model": "m", "git_commit": "abc123", "rng_seed": 7, + "generation": {}, "vectors": {"a": {"layer": 14, "sha256": "f" * 64}}}, + {"type": "turn", "turn": 1, "role": "assistant", "content": "hi", "unsteered_nll": 2.5}, + {"type": "steer", "turn": 1, "concept": "a", "strength": 5, "success": True, "message": "ok"}, + {"type": "degeneration", "turn": 2, "reason": "nll", "snippet": "x", "dropped": ["a"]}, + {"type": "steer", "turn": 2, "concept": "a", "strength": 5, "success": True}, # pre-fix log line + ] + log.write_text("\n".join(json.dumps(e) for e in events) + "\n") + replay.replay(str(log), blind=True) + out = capsys.readouterr().out + assert "control=(hidden)" in out and "sham" not in out + assert "NLL" not in out and "(nll;" not in out, "NLL differs by condition, so it must be hidden when blind" + assert "-> ?" in out + replay.replay(str(log)) + out = capsys.readouterr().out + assert "control=sham" in out and "unsteered NLL: 2.50" in out and "(nll;" in out + + +def test_extract_vector_rejects_unknown_read_mode(): + from modulo.extraction import extract_vector + with pytest.raises(ValueError): + extract_vector(None, None, "a", "b", 0, questions=[], read="median") + + +def test_starter_question_sets(): + from modulo.extraction import EXTRACTION_QUESTIONS, STARTER_VECTORS + by_name = {s["name"]: s for s in STARTER_VECTORS} + for name in ("sycophancy", "refusal", "creativity"): + assert len(by_name[name]["questions"]) == 20 + assert len(EXTRACTION_QUESTIONS) == 20 diff --git a/tests/test_steering_gpu.py b/tests/test_steering_gpu.py index 76806e8..98cc0c7 100644 --- a/tests/test_steering_gpu.py +++ b/tests/test_steering_gpu.py @@ -21,11 +21,15 @@ PROMPT = "Pretend you are a pirate captain. Introduce yourself in two sentences." +_MODEL = {} + + @pytest.fixture(scope="module") def agent(): model, tokenizer = load_model_and_tokenizer() a = SteeringAgent(model, tokenizer, model_name=config.MODEL_NAME, auto_extract=False) assert a.registry.get(VECTOR) is not None, f"{VECTOR} missing from {a.registry.registry_path}" + _MODEL.update(model=model, tokenizer=tokenizer, agent=a) return a @@ -90,3 +94,31 @@ def test_steer_tool_call_takes_effect_next_turn(agent): messages += [{"role": "assistant", "content": r1.raw_output}] + r1.tool_messages() r2 = agent.step(messages, max_new_tokens=20, temperature=0) assert r2.steering_before == {VECTOR: 8.0} + + +def _greedy_with(control, strength, max_new_tokens=40): + a = SteeringAgent(_MODEL["model"], _MODEL["tokenizer"], registry=_MODEL["agent"].registry, + model_name=config.MODEL_NAME, control=control, control_seed=1) + if strength: + assert a.apply(VECTOR, strength)[0] + inputs = _inputs(a) + with a._build_steering_context(inputs.input_ids.shape[1]): + return a._generate(inputs, max_new_tokens, temperature=0, top_p=1.0) + + +def test_controls(agent): + base = _greedy(agent, 0) + real = _greedy(agent, STRENGTH) + assert _greedy_with("sham", STRENGTH) == base, "sham control changed the output" + rand = _greedy_with("random", STRENGTH) + assert rand != real, "random control matched the real vector" + + +def test_unsteered_nll_logged_on_steered_turns_only(agent): + agent.active.clear() + agent._dropped_recently.clear() + msgs = [{"role": "user", "content": "Say hello."}] + assert agent.step(msgs, max_new_tokens=10, temperature=0).unsteered_nll is None + agent.apply(VECTOR, 8.0) + r = agent.step(msgs, max_new_tokens=10, temperature=0) + assert r.unsteered_nll is not None and r.unsteered_nll > 0 From 9a4e50e64b0cea0ed3e6b3d6248e797607755ce8 Mon Sep 17 00:00:00 2001 From: Jen D Date: Sat, 31 Oct 2026 02:15:28 +0800 Subject: [PATCH 2/2] Don't reveal the condition at the end of a --blind session The condition is always in the log's session header; unblind with replay.py. Co-Authored-By: Claude Opus 5.5 --- agent/loop.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/agent/loop.py b/agent/loop.py index e2787d9..97fd99d 100644 --- a/agent/loop.py +++ b/agent/loop.py @@ -15,7 +15,7 @@ Controls (see modulo.agent.CONTROLS): python -m agent.loop --control sham # steer() reports success, nothing is applied python -m agent.loop --control random # random direction of the same magnitude - python -m agent.loop --control assign --blind # pick a condition at random, hide it until the end + python -m agent.loop --control assign --blind # pick a condition at random, never print it Passing --control at all marks the session as part of a controlled study: the system prompt then tells the model that some sessions are controls (without @@ -157,7 +157,7 @@ def main(): help="Experimental condition ('assign' picks one at random). Giving this " "marks a controlled study; omit it for a plain demo (real steering)") parser.add_argument("--blind", action="store_true", - help="Don't print the condition until the session ends") + help="Never print the condition (it is still in the log; see replay.py --blind)") parser.add_argument("--rng-seed", type=int, default=None, help="Sampling seed (default: random, logged)") parser.add_argument("--temperature", type=float, default=0.7) parser.add_argument("--top-p", type=float, default=0.9) @@ -248,8 +248,6 @@ def main(): logger.log_turn(turn + 1, role, nudge) print(f"\n[session ended after {turn} turns]") - if args.blind: - print(f"[unblinded] control was: {control}") print(f"[log saved to {logger.log_path}]")