From 0aabaac08c24949469f213917a646b76a4207afd Mon Sep 17 00:00:00 2001 From: Jen D Date: Mon, 2 Nov 2026 10:45:22 +0800 Subject: [PATCH] Stop the system prompt anchoring the model's choices Models copied the prompt's example call: Mistral called steer(sycophancy, 5.0) nearly every turn, the example word for word, and the median strength chosen was 5 for three of four models. - The steer, drop, report and create examples use placeholders (, , , ), and the strength and decay explanations no longer use a strength a model could copy. - A call naming one of Modulo's tools that isn't valid JSON (a copied placeholder, a missing brace) is reported to the model instead of being silently ignored. Valid JSON that isn't a call, such as an echoed tool result, is left alone. - The vector menu is shuffled once per session (menu_seed; the demo uses --rng-seed, and logs menu_seed and menu_order), since sycophancy was always listed first. Vectors made with create join the end. Checked on a GPU before the shuffle (12 sessions): Llama's strengths spread from -5 to +10 and no model favoured sycophancy, but Mistral copies placeholders verbatim and the feedback doesn't change that. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 11 +++++-- agent/loop.py | 3 ++ modulo/agent.py | 66 +++++++++++++++++++++++++++++++------- modulo/registry.py | 6 ++-- tests/test_agent_logic.py | 56 +++++++++++++++++++++++++++++++- tests/test_steering_gpu.py | 3 +- 6 files changed, 127 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a6aa8fb..10e5fa7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,6 +182,9 @@ agent = SteeringAgent(model, tokenizer, auto_extract=False) # Also offer unverified community-index vectors agent = SteeringAgent(model, tokenizer, include_community=True) +# Fix the order the vectors are listed in (default: shuffled per session, kept in agent.menu_seed) +agent = SteeringAgent(model, tokenizer, menu_seed=42) + # Controlled study: sham or random-direction condition, disclosed to the model agent = SteeringAgent(model, tokenizer, control="sham", disclose_controls=True) @@ -239,7 +242,7 @@ python replay.py logs/session_*.jsonl # then unblind Each `logs/session_*.jsonl` starts with a `session` record: - argv, git commit and dirty flag, model, dtype, torch/transformers versions -- control condition, RNG seed (`--rng-seed`, random if omitted), generation settings (`--temperature`, `--top-p`, `--max-new-tokens`), any `--steer` at start +- control condition, RNG seed (`--rng-seed`, random if omitted; it also seeds the vector menu order, logged as `menu_seed` and `menu_order`), generation settings (`--temperature`, `--top-p`, `--max-new-tokens`), any `--steer` at start - safety-rail constants, and each vector's registry entry plus file sha256 Assistant turns also record the raw output, the exact system prompt, and `unsteered_nll`. @@ -257,13 +260,17 @@ NLL measures "unusual for this model", and strong steering is unusual by design. ## The tools the model has -The model emits these as JSON objects anywhere in its output (bare or inside `` tags): +The model emits these as JSON objects anywhere in its output (bare or inside `` tags) (the values below are illustrations; the model's own prompt shows ``, `` and `` placeholders instead, because models copied the example values: Mistral called `steer(sycophancy, 5.0)`, the old example, nearly every turn): - `{"name": "steer", "arguments": {"concept": "sycophancy", "strength": 5.0}}` - `{"name": "drop", "arguments": {"concept": "sycophancy"}}` - `{"name": "report", "arguments": {"concept": "sycophancy", "worked": true, "notes": "..."}}` - `{"name": "create", "arguments": {"concept": "new_name", "more": "...", "less": "..."}}` +The vectors are listed in an order shuffled once per session (`menu_seed`; the demo uses `--rng-seed`), since models tend to pick the first one listed: with `sycophancy` first in every session, Mistral chose it whenever it steered. Vectors made with `create` join the end of the list. + +A call that names one of these tools but isn't valid JSON (a copied ``, a missing brace) isn't run, and the model is told why in its tool results, instead of being silently ignored. + `report` is only accepted once the vector has stopped steering the model (it expired or was dropped) and at least one unsteered turn has followed. A model being steered tends to become the state instead of noticing it, so it reports by looking back at its steered turns. Once a vector has ended, `agent.status()` (which appears in the system prompt and the `--auto` status message) lists it as one the model can report on, until it does. `agent.debrief()` ends a session's steering: it drops every vector, refuses further `steer` and `create` calls, and returns a status message listing the vectors the model can report on (or `None` if there are none). Send it as a `tool` message and give the model one more turn. The demo does this after the last `--auto` turn (`--no-debrief` to skip it). In testing, OLMo-2 mostly kept the steered voice through the unsteered debrief turn, because its own steered turns are in the context, so a debrief report is still not a sober one. diff --git a/agent/loop.py b/agent/loop.py index ed2c872..255c779 100644 --- a/agent/loop.py +++ b/agent/loop.py @@ -87,6 +87,8 @@ def session_metadata(agent: SteeringAgent, args, rng_seed: int, gen_params: dict "transformers": transformers.__version__, "control": agent.control, "control_seed": agent.control_seed, + "menu_seed": agent.menu_seed, + "menu_order": agent.menu_order(), "controlled_study": agent.disclose_controls, "blind": args.blind, "rng_seed": rng_seed, @@ -206,6 +208,7 @@ def main(): # Offer every vector we can: unverified index entries too (marked as such in the prompt) agent = SteeringAgent(model, tokenizer, model_name=args.model, include_community=True, control=control, control_seed=rng_seed, disclose_controls=study, + menu_seed=rng_seed, nll_threshold=args.nll_threshold) for name, strength in steers: ok, msg = agent.apply(name, strength) diff --git a/modulo/agent.py b/modulo/agent.py index 11193a0..7f4a9f4 100644 --- a/modulo/agent.py +++ b/modulo/agent.py @@ -13,6 +13,7 @@ """ import json import math +import random import re import zlib from collections import Counter @@ -161,6 +162,25 @@ def _find_tool_calls(text: str) -> list[tuple[int, int, ToolCall]]: return found +_TOOL_NAME_AT = re.compile(r'\{\s*"name"\s*:\s*"(steer|drop|report|create)"') + + +def malformed_tool_calls(text: str) -> list[tuple[str, str]]: + """(tool name, snippet) for each call to one of Modulo's tools that isn't valid JSON, + such as a copied placeholder: {"name": "steer", "arguments": {"strength": }}.""" + spans = [(start, end) for start, end, _ in _find_tool_calls(text)] + found = [] + for m in _TOOL_NAME_AT.finditer(text): + if any(start <= m.start() < end for start, end in spans): + continue + try: + _DECODER.raw_decode(text, m.start()) + continue # valid JSON but not a call, e.g. an echoed tool result + except json.JSONDecodeError: + found.append((m.group(1), text[m.start():m.start() + 120])) + return found + + def parse_tool_calls(text: str) -> list[ToolCall]: return [tc for _, _, tc in _find_tool_calls(text)] @@ -297,35 +317,37 @@ def __exit__(self, *exc): You can call steer by emitting a JSON object on its own line, like this: -{{"name": "steer", "arguments": {{"concept": "{example}", "strength": 5.0}}}} +{{"name": "steer", "arguments": {{"concept": "", "strength": }}}} + +with replaced by one of the vectors listed below and by a strength. Parameters: - concept: one of the available steering vectors (listed below) - strength: a number; positive or negative. It is the size of the push: \ {full_strength:g} adds a vector as large as your typical activations at that \ -layer, so 5.0 adds one {five_pct:.0f}% as large. Clamped to each vector's range. +layer, so 1 adds one about {one_pct:.0f}% as large. Clamped to each vector's range. # How it works - When you call steer, the vector is applied from your NEXT turn and stays \ active until it decays away. - Each turn after the first, all active vectors are multiplied by {decay}. \ -For example, strength 10 is about {ten_after_10:.1f} after 10 turns and is \ +So about {left_after_10:.0%} of a vector's strength is left after 10 turns, and it is \ removed once it falls below {floor}. - You can have up to {max_active} vectors active at once. - You can refresh a vector (call steer again with the same concept) to reset \ its strength. After you drop a vector you cannot re-apply it for the next \ {cooldown} turns. -- You can also call: {{"name": "drop", "arguments": {{"concept": "{example}"}}}} \ +- You can also call: {{"name": "drop", "arguments": {{"concept": ""}}}} \ to remove a vector immediately. - Once a vector has stopped steering you (it decayed away or you dropped it), \ -you can call: {{"name": "report", "arguments": {{"concept": "{example}", \ -"worked": true, "notes": "..."}}}} to log whether it had its intended effect. \ +you can call: {{"name": "report", "arguments": {{"concept": "", \ +"worked": , "notes": "..."}}}} to log whether it had its intended effect. \ Reports aren't accepted while the vector is active, because a state is hard to \ judge from inside it: look back at the turns it steered and compare them with \ your turns since. This is for research. - You can make a new steering vector: {{"name": "create", "arguments": \ -{{"concept": "new_name", "more": "...", "less": "..."}}}}. `more` and `less` are \ +{{"concept": "", "more": "...", "less": "..."}}}}. `more` and `less` are \ two system prompts, each written to an assistant ("You are ..."), for the two ends \ of the direction you want. Modulo has you answer a fixed set of questions under \ each one and takes the difference between your activations, which takes a few \ @@ -366,7 +388,7 @@ def __init__(self, model, tokenizer, registry: Optional[VectorRegistry] = None, model_name: Optional[str] = None, auto_extract: bool = True, vectors_dir: Optional[Path] = None, include_community: bool = False, control: str = "real", control_seed: int = 0, disclose_controls: bool = False, - nll_threshold: Optional[float] = None): + nll_threshold: Optional[float] = None, menu_seed: Optional[int] = None): """ Args: model: HF causal LM (must have model.model.layers) @@ -387,12 +409,18 @@ def __init__(self, model, tokenizer, registry: Optional[VectorRegistry] = None, nll_threshold: if set, a steered output whose unsteered mean NLL exceeds this is treated as degenerate. Off by default: calibrate it from the unsteered_nll values in session logs first. + menu_seed: seed for the order the vectors are listed in the prompt, which is + shuffled once per session so no vector is always first (models + tend to pick the first one). None: a random order, kept in + self.menu_seed so a session can be reproduced. """ if control not in CONTROLS: raise ValueError(f"control must be one of {CONTROLS}, got {control!r}") self.control = control self.control_seed = control_seed self.disclose_controls = disclose_controls + self.menu_seed = menu_seed if menu_seed is not None else random.SystemRandom().randrange(2**31) + self._menu_order: Optional[list[str]] = None # fixed at the first prompt of the session self.nll_threshold = nll_threshold self.model = model self.tokenizer = tokenizer @@ -434,18 +462,17 @@ def build_system_prompt(self) -> str: names = self.registry.list_names() return SYSTEM_PROMPT_TEMPLATE.format( reality_clause=CONTROLLED_REALITY_CLAUSE if self.disclose_controls else REALITY_CLAUSE, - example=names[0] if names else "concept_name", decay=DECAY_FACTOR, - ten_after_10=10 * DECAY_FACTOR ** 9, + left_after_10=DECAY_FACTOR ** 9, floor=STRENGTH_FLOOR, full_strength=FULL_TOKEN_STRENGTH, - five_pct=500 / FULL_TOKEN_STRENGTH, + one_pct=100 / FULL_TOKEN_STRENGTH, max_active=MAX_ACTIVE_VECTORS, max_created=MAX_CREATED_VECTORS, created_range=CREATED_RANGE, cooldown=COOLDOWN_TURNS, vector_descriptions=self.registry.describe_for_prompt( - max_strength=MAX_STRENGTH), + max_strength=MAX_STRENGTH, order=self.menu_order()), steering_status=self.status(), ) @@ -466,6 +493,17 @@ def reportable(self) -> list[str]: return [n for n, (_, t) in self._last_steered.items() if self._report_refusal(n) is None and self._reported_turn.get(n, -1) <= t] + def menu_order(self) -> list[str]: + """The vectors in the order the prompt lists them: shuffled once per session + (menu_seed), with vectors added later (create) appended in the order they came.""" + names = self.registry.list_names() + if self._menu_order is None: + self._menu_order = sorted(names) + random.Random(self.menu_seed).shuffle(self._menu_order) + self._menu_order = [n for n in self._menu_order if n in names] + self._menu_order += [n for n in names if n not in self._menu_order] + return list(self._menu_order) + def status(self) -> str: lines = [] for name, av in self.active.items(): @@ -787,6 +825,10 @@ def step(self, messages: list[dict], max_new_tokens: int = 300, tool_calls = parse_tool_calls(raw_output) prose = strip_tool_calls(raw_output) tool_results = [self._execute(tc) for tc in tool_calls] + tool_results += [ToolResult(name, {}, False, + f"This {name} call isn't valid JSON, so it wasn't run: {snippet!r}. Write it " + f"as one JSON object, with real values in place of any .") + for name, snippet in malformed_tool_calls(raw_output)] self.decay() steering_after = {n: av.current_strength for n, av in self.active.items()} diff --git a/modulo/registry.py b/modulo/registry.py index b804461..891ccc5 100644 --- a/modulo/registry.py +++ b/modulo/registry.py @@ -129,11 +129,13 @@ def add_report(self, report: dict): def vector_sha256(self, entry: VectorEntry) -> Optional[str]: return file_sha256(Path(self.vectors_dir) / entry.file) - def describe_for_prompt(self, max_strength: float = float("inf")) -> str: + def describe_for_prompt(self, max_strength: float = float("inf"), order: Optional[list[str]] = None) -> str: + """One line per vector, in `order` (names) if given, else registry order.""" if not self.entries: return "(no steering vectors available — use extract_auto() to generate some)" lines = [] - for name, e in self.entries.items(): + for name in (order if order is not None else list(self.entries)): + e = self.entries[name] lo = max(e.safe_strength_min, -max_strength) hi = min(e.safe_strength_max, max_strength) lines.append( diff --git a/tests/test_agent_logic.py b/tests/test_agent_logic.py index 9f56811..e9b3bf1 100644 --- a/tests/test_agent_logic.py +++ b/tests/test_agent_logic.py @@ -506,7 +506,7 @@ def rand(seed): def test_system_prompt_identical_across_controls_when_disclosed(): - prompts = {c: SteeringAgent(None, None, registry=StubRegistry(), control=c, + prompts = {c: SteeringAgent(None, None, registry=StubRegistry(), control=c, menu_seed=7, disclose_controls=True).build_system_prompt() for c in A.CONTROLS} assert len(set(prompts.values())) == 1 assert "experimental controls" in prompts["real"] @@ -666,3 +666,57 @@ def test_no_creating_after_the_debrief(creator): creator.debrief() r = creator._execute(ToolCall("create", {"concept": "late", "more": "x", "less": "y"}, "")) assert not r.ok and "session is over" in r.message and not creator._calls + + +# --- the prompt's examples don't suggest values --- + +def test_prompt_examples_use_placeholders_not_values(agent): + prompt = agent.build_system_prompt() + assert '"concept": "", "strength": ' in prompt + assert '"worked": ' in prompt + assert '"concept": "a"' not in prompt and '"strength": 5' not in prompt and "5.0" not in prompt + assert "1 adds one about 7% as large" in prompt and "about 23% of a vector's strength" in prompt + + +def test_copied_placeholders_get_feedback_instead_of_silence(agent, monkeypatch): + out = _steered_step(agent, monkeypatch, 'Trying: {"name": "steer", "arguments": {"concept": "a", "strength": }} ' + 'and {"name": "drop", "arguments": {"concept": "b"}}') + by_name = {r.name: r for r in out.tool_results} + assert not by_name["drop"].ok # the valid call still ran ('b' isn't active) + assert not by_name["steer"].ok and "isn't valid JSON" in by_name["steer"].message + assert "" in by_name["steer"].message and agent.active == {} + assert len(out.tool_results) == 2 + msgs = out.tool_messages() + assert any("isn't valid JSON" in m["content"] for m in msgs) + + +def test_echoed_tool_results_are_not_reported_as_malformed_calls(): + from modulo.agent import malformed_tool_calls + echo = '{"name": "steer", "ok": true, "message": "steer(a, +5.00) applied"}' + assert malformed_tool_calls(echo) == [] + assert [n for n, _ in malformed_tool_calls('{"name": "report", "arguments": {"worked": }}')] == ["report"] + + +# --- the menu order is shuffled per session --- + +def _menu(agent): + prompt = agent.build_system_prompt() + menu = prompt.split("# Available steering vectors")[1].split("# Important notes")[0] + return [line[2:].split(":")[0] for line in menu.strip().splitlines()] + + +def test_menu_order_is_shuffled_per_session_and_seeded(): + names = [f"v{i}" for i in range(8)] + + def order(seed): + return _menu(SteeringAgent(None, None, registry=StubRegistry(names=names), menu_seed=seed)) + assert order(1) == order(1) and sorted(order(1)) == names + assert len({tuple(order(s)) for s in range(6)}) > 1 # different sessions see different orders + a = SteeringAgent(None, None, registry=StubRegistry(names=names)) + assert isinstance(a.menu_seed, int) and _menu(a) == _menu(a) # stable within a session + + +def test_vectors_created_mid_session_join_the_end_of_the_menu(creator): + before = _menu(creator) + creator._execute(ToolCall("create", {"concept": "wonder", "more": "x", "less": "y"}, "")) + assert _menu(creator) == before + ["wonder"] diff --git a/tests/test_steering_gpu.py b/tests/test_steering_gpu.py index 4c985b9..aa7e65e 100644 --- a/tests/test_steering_gpu.py +++ b/tests/test_steering_gpu.py @@ -97,7 +97,8 @@ def test_steer_tool_call_takes_effect_next_turn(agent): def _greedy_with(control, strength, max_new_tokens=40): a = SteeringAgent(_MODEL["model"], _MODEL["tokenizer"], registry=_MODEL["agent"].registry, - model_name=config.MODEL_NAME, control=control, control_seed=1) + model_name=config.MODEL_NAME, control=control, control_seed=1, + menu_seed=_MODEL["agent"].menu_seed) # same prompt as the fixture agent if strength: assert a.apply(VECTOR, strength)[0] inputs = _inputs(a)