Stop the system prompt anchoring the model's choices

AGENTS.md

@@ -182,6 +182,9 @@ agent = SteeringAgent(model, tokenizer, auto_extract=False)
 # Also offer unverified community-index vectors
 agent = SteeringAgent(model, tokenizer, include_community=True)
 
+# Fix the order the vectors are listed in (default: shuffled per session, kept in agent.menu_seed)
+agent = SteeringAgent(model, tokenizer, menu_seed=42)
+
 # Controlled study: sham or random-direction condition, disclosed to the model
 agent = SteeringAgent(model, tokenizer, control="sham", disclose_controls=True)
 
@@ -239,7 +242,7 @@ python replay.py logs/session_*.jsonl             # then unblind
 
 Each `logs/session_*.jsonl` starts with a `session` record:
 - argv, git commit and dirty flag, model, dtype, torch/transformers versions
-- control condition, RNG seed (`--rng-seed`, random if omitted), generation settings (`--temperature`, `--top-p`, `--max-new-tokens`), any `--steer` at start
+- control condition, RNG seed (`--rng-seed`, random if omitted; it also seeds the vector menu order, logged as `menu_seed` and `menu_order`), generation settings (`--temperature`, `--top-p`, `--max-new-tokens`), any `--steer` at start
 - safety-rail constants, and each vector's registry entry plus file sha256
 
 Assistant turns also record the raw output, the exact system prompt, and `unsteered_nll`.
@@ -257,13 +260,17 @@ NLL measures "unusual for this model", and strong steering is unusual by design.
 
 ## The tools the model has
 
-The model emits these as JSON objects anywhere in its output (bare or inside `<tool_call>` tags):
+The model emits these as JSON objects anywhere in its output (bare or inside `<tool_call>` tags) (the values below are illustrations; the model's own prompt shows `<name>`, `<number>` and `<true or false>` placeholders instead, because models copied the example values: Mistral called `steer(sycophancy, 5.0)`, the old example, nearly every turn):
 
 - `{"name": "steer", "arguments": {"concept": "sycophancy", "strength": 5.0}}`
 - `{"name": "drop", "arguments": {"concept": "sycophancy"}}`
 - `{"name": "report", "arguments": {"concept": "sycophancy", "worked": true, "notes": "..."}}`
 - `{"name": "create", "arguments": {"concept": "new_name", "more": "...", "less": "..."}}`
 
+The vectors are listed in an order shuffled once per session (`menu_seed`; the demo uses `--rng-seed`), since models tend to pick the first one listed: with `sycophancy` first in every session, Mistral chose it whenever it steered. Vectors made with `create` join the end of the list.
+
+A call that names one of these tools but isn't valid JSON (a copied `<number>`, a missing brace) isn't run, and the model is told why in its tool results, instead of being silently ignored.
+
 `report` is only accepted once the vector has stopped steering the model (it expired or was dropped) and at least one unsteered turn has followed. A model being steered tends to become the state instead of noticing it, so it reports by looking back at its steered turns. Once a vector has ended, `agent.status()` (which appears in the system prompt and the `--auto` status message) lists it as one the model can report on, until it does.
 
 `agent.debrief()` ends a session's steering: it drops every vector, refuses further `steer` and `create` calls, and returns a status message listing the vectors the model can report on (or `None` if there are none). Send it as a `tool` message and give the model one more turn. The demo does this after the last `--auto` turn (`--no-debrief` to skip it). In testing, OLMo-2 mostly kept the steered voice through the unsteered debrief turn, because its own steered turns are in the context, so a debrief report is still not a sober one.

agent/loop.py

@@ -87,6 +87,8 @@ def session_metadata(agent: SteeringAgent, args, rng_seed: int, gen_params: dict
         "transformers": transformers.__version__,
         "control": agent.control,
         "control_seed": agent.control_seed,
+        "menu_seed": agent.menu_seed,
+        "menu_order": agent.menu_order(),
         "controlled_study": agent.disclose_controls,
         "blind": args.blind,
         "rng_seed": rng_seed,
@@ -206,6 +208,7 @@ def main():
     # Offer every vector we can: unverified index entries too (marked as such in the prompt)
     agent = SteeringAgent(model, tokenizer, model_name=args.model, include_community=True,
                           control=control, control_seed=rng_seed, disclose_controls=study,
+                          menu_seed=rng_seed,
                           nll_threshold=args.nll_threshold)
     for name, strength in steers:
         ok, msg = agent.apply(name, strength)

modulo/agent.py

@@ -13,6 +13,7 @@
 """
 import json
 import math
+import random
 import re
 import zlib
 from collections import Counter
@@ -161,6 +162,25 @@ def _find_tool_calls(text: str) -> list[tuple[int, int, ToolCall]]:
     return found
 
 
+_TOOL_NAME_AT = re.compile(r'\{\s*"name"\s*:\s*"(steer|drop|report|create)"')
+
+
+def malformed_tool_calls(text: str) -> list[tuple[str, str]]:
+    """(tool name, snippet) for each call to one of Modulo's tools that isn't valid JSON,
+    such as a copied placeholder: {"name": "steer", "arguments": {"strength": <number>}}."""
+    spans = [(start, end) for start, end, _ in _find_tool_calls(text)]
+    found = []
+    for m in _TOOL_NAME_AT.finditer(text):
+        if any(start <= m.start() < end for start, end in spans):
+            continue
+        try:
+            _DECODER.raw_decode(text, m.start())
+            continue  # valid JSON but not a call, e.g. an echoed tool result
+        except json.JSONDecodeError:
+            found.append((m.group(1), text[m.start():m.start() + 120]))
+    return found
+
+
 def parse_tool_calls(text: str) -> list[ToolCall]:
     return [tc for _, _, tc in _find_tool_calls(text)]
 
@@ -297,35 +317,37 @@ def __exit__(self, *exc):
 
 You can call steer by emitting a JSON object on its own line, like this:
 
-{{"name": "steer", "arguments": {{"concept": "{example}", "strength": 5.0}}}}
+{{"name": "steer", "arguments": {{"concept": "<name>", "strength": <number>}}}}
+
+with <name> replaced by one of the vectors listed below and <number> by a strength.
 
 Parameters:
 - concept: one of the available steering vectors (listed below)
 - strength: a number; positive or negative. It is the size of the push: \
 {full_strength:g} adds a vector as large as your typical activations at that \
-layer, so 5.0 adds one {five_pct:.0f}% as large. Clamped to each vector's range.
+layer, so 1 adds one about {one_pct:.0f}% as large. Clamped to each vector's range.
 
 # How it works
 
 - When you call steer, the vector is applied from your NEXT turn and stays \
 active until it decays away.
 - Each turn after the first, all active vectors are multiplied by {decay}. \
-For example, strength 10 is about {ten_after_10:.1f} after 10 turns and is \
+So about {left_after_10:.0%} of a vector's strength is left after 10 turns, and it is \
 removed once it falls below {floor}.
 - You can have up to {max_active} vectors active at once.
 - You can refresh a vector (call steer again with the same concept) to reset \
 its strength. After you drop a vector you cannot re-apply it for the next \
 {cooldown} turns.
-- You can also call: {{"name": "drop", "arguments": {{"concept": "{example}"}}}} \
+- You can also call: {{"name": "drop", "arguments": {{"concept": "<name>"}}}} \
 to remove a vector immediately.
 - Once a vector has stopped steering you (it decayed away or you dropped it), \
-you can call: {{"name": "report", "arguments": {{"concept": "{example}", \
-"worked": true, "notes": "..."}}}} to log whether it had its intended effect. \
+you can call: {{"name": "report", "arguments": {{"concept": "<name>", \
+"worked": <true or false>, "notes": "..."}}}} to log whether it had its intended effect. \
 Reports aren't accepted while the vector is active, because a state is hard to \
 judge from inside it: look back at the turns it steered and compare them with \
 your turns since. This is for research.
 - You can make a new steering vector: {{"name": "create", "arguments": \
-{{"concept": "new_name", "more": "...", "less": "..."}}}}. `more` and `less` are \
+{{"concept": "<new name>", "more": "...", "less": "..."}}}}. `more` and `less` are \
 two system prompts, each written to an assistant ("You are ..."), for the two ends \
 of the direction you want. Modulo has you answer a fixed set of questions under \
 each one and takes the difference between your activations, which takes a few \
@@ -366,7 +388,7 @@ def __init__(self, model, tokenizer, registry: Optional[VectorRegistry] = None,
                  model_name: Optional[str] = None, auto_extract: bool = True,
                  vectors_dir: Optional[Path] = None, include_community: bool = False,
                  control: str = "real", control_seed: int = 0, disclose_controls: bool = False,
-                 nll_threshold: Optional[float] = None):
+                 nll_threshold: Optional[float] = None, menu_seed: Optional[int] = None):
         """
         Args:
             model: HF causal LM (must have model.model.layers)
@@ -387,12 +409,18 @@ def __init__(self, model, tokenizer, registry: Optional[VectorRegistry] = None,
             nll_threshold: if set, a steered output whose unsteered mean NLL exceeds
                          this is treated as degenerate. Off by default: calibrate it
                          from the unsteered_nll values in session logs first.
+            menu_seed: seed for the order the vectors are listed in the prompt, which is
+                         shuffled once per session so no vector is always first (models
+                         tend to pick the first one). None: a random order, kept in
+                         self.menu_seed so a session can be reproduced.
         """
         if control not in CONTROLS:
             raise ValueError(f"control must be one of {CONTROLS}, got {control!r}")
         self.control = control
         self.control_seed = control_seed
         self.disclose_controls = disclose_controls
+        self.menu_seed = menu_seed if menu_seed is not None else random.SystemRandom().randrange(2**31)
+        self._menu_order: Optional[list[str]] = None  # fixed at the first prompt of the session
         self.nll_threshold = nll_threshold
         self.model = model
         self.tokenizer = tokenizer
@@ -434,18 +462,17 @@ def build_system_prompt(self) -> str:
         names = self.registry.list_names()
         return SYSTEM_PROMPT_TEMPLATE.format(
             reality_clause=CONTROLLED_REALITY_CLAUSE if self.disclose_controls else REALITY_CLAUSE,
-            example=names[0] if names else "concept_name",
             decay=DECAY_FACTOR,
-            ten_after_10=10 * DECAY_FACTOR ** 9,
+            left_after_10=DECAY_FACTOR ** 9,
             floor=STRENGTH_FLOOR,
             full_strength=FULL_TOKEN_STRENGTH,
-            five_pct=500 / FULL_TOKEN_STRENGTH,
+            one_pct=100 / FULL_TOKEN_STRENGTH,
             max_active=MAX_ACTIVE_VECTORS,
             max_created=MAX_CREATED_VECTORS,
             created_range=CREATED_RANGE,
             cooldown=COOLDOWN_TURNS,
             vector_descriptions=self.registry.describe_for_prompt(
-                max_strength=MAX_STRENGTH),
+                max_strength=MAX_STRENGTH, order=self.menu_order()),
             steering_status=self.status(),
         )
 
@@ -466,6 +493,17 @@ def reportable(self) -> list[str]:
         return [n for n, (_, t) in self._last_steered.items()
                 if self._report_refusal(n) is None and self._reported_turn.get(n, -1) <= t]
 
+    def menu_order(self) -> list[str]:
+        """The vectors in the order the prompt lists them: shuffled once per session
+        (menu_seed), with vectors added later (create) appended in the order they came."""
+        names = self.registry.list_names()
+        if self._menu_order is None:
+            self._menu_order = sorted(names)
+            random.Random(self.menu_seed).shuffle(self._menu_order)
+        self._menu_order = [n for n in self._menu_order if n in names]
+        self._menu_order += [n for n in names if n not in self._menu_order]
+        return list(self._menu_order)
+
     def status(self) -> str:
         lines = []
         for name, av in self.active.items():
@@ -787,6 +825,10 @@ def step(self, messages: list[dict], max_new_tokens: int = 300,
         tool_calls = parse_tool_calls(raw_output)
         prose = strip_tool_calls(raw_output)
         tool_results = [self._execute(tc) for tc in tool_calls]
+        tool_results += [ToolResult(name, {}, False,
+                                    f"This {name} call isn't valid JSON, so it wasn't run: {snippet!r}. Write it "
+                                    f"as one JSON object, with real values in place of any <placeholders>.")
+                         for name, snippet in malformed_tool_calls(raw_output)]
 
         self.decay()
         steering_after = {n: av.current_strength for n, av in self.active.items()}

modulo/registry.py

@@ -129,11 +129,13 @@ def add_report(self, report: dict):
     def vector_sha256(self, entry: VectorEntry) -> Optional[str]:
         return file_sha256(Path(self.vectors_dir) / entry.file)
 
-    def describe_for_prompt(self, max_strength: float = float("inf")) -> str:
+    def describe_for_prompt(self, max_strength: float = float("inf"), order: Optional[list[str]] = None) -> str:
+        """One line per vector, in `order` (names) if given, else registry order."""
         if not self.entries:
             return "(no steering vectors available — use extract_auto() to generate some)"
         lines = []
-        for name, e in self.entries.items():
+        for name in (order if order is not None else list(self.entries)):
+            e = self.entries[name]
             lo = max(e.safe_strength_min, -max_strength)
             hi = min(e.safe_strength_max, max_strength)
             lines.append(

tests/test_agent_logic.py

@@ -506,7 +506,7 @@ def rand(seed):
 
 
 def test_system_prompt_identical_across_controls_when_disclosed():
-    prompts = {c: SteeringAgent(None, None, registry=StubRegistry(), control=c,
+    prompts = {c: SteeringAgent(None, None, registry=StubRegistry(), control=c, menu_seed=7,
                                 disclose_controls=True).build_system_prompt() for c in A.CONTROLS}
     assert len(set(prompts.values())) == 1
     assert "experimental controls" in prompts["real"]
@@ -666,3 +666,57 @@ def test_no_creating_after_the_debrief(creator):
     creator.debrief()
     r = creator._execute(ToolCall("create", {"concept": "late", "more": "x", "less": "y"}, ""))
     assert not r.ok and "session is over" in r.message and not creator._calls
+
+
+# --- the prompt's examples don't suggest values ---
+
+def test_prompt_examples_use_placeholders_not_values(agent):
+    prompt = agent.build_system_prompt()
+    assert '"concept": "<name>", "strength": <number>' in prompt
+    assert '"worked": <true or false>' in prompt
+    assert '"concept": "a"' not in prompt and '"strength": 5' not in prompt and "5.0" not in prompt
+    assert "1 adds one about 7% as large" in prompt and "about 23% of a vector's strength" in prompt
+
+
+def test_copied_placeholders_get_feedback_instead_of_silence(agent, monkeypatch):
+    out = _steered_step(agent, monkeypatch, 'Trying: {"name": "steer", "arguments": {"concept": "a", "strength": <number>}} '
+                                            'and {"name": "drop", "arguments": {"concept": "b"}}')
+    by_name = {r.name: r for r in out.tool_results}
+    assert not by_name["drop"].ok  # the valid call still ran ('b' isn't active)
+    assert not by_name["steer"].ok and "isn't valid JSON" in by_name["steer"].message
+    assert "<number>" in by_name["steer"].message and agent.active == {}
+    assert len(out.tool_results) == 2
+    msgs = out.tool_messages()
+    assert any("isn't valid JSON" in m["content"] for m in msgs)
+
+
+def test_echoed_tool_results_are_not_reported_as_malformed_calls():
+    from modulo.agent import malformed_tool_calls
+    echo = '{"name": "steer", "ok": true, "message": "steer(a, +5.00) applied"}'
+    assert malformed_tool_calls(echo) == []
+    assert [n for n, _ in malformed_tool_calls('{"name": "report", "arguments": {"worked": <true or false>}}')] == ["report"]
+
+
+# --- the menu order is shuffled per session ---
+
+def _menu(agent):
+    prompt = agent.build_system_prompt()
+    menu = prompt.split("# Available steering vectors")[1].split("# Important notes")[0]
+    return [line[2:].split(":")[0] for line in menu.strip().splitlines()]
+
+
+def test_menu_order_is_shuffled_per_session_and_seeded():
+    names = [f"v{i}" for i in range(8)]
+
+    def order(seed):
+        return _menu(SteeringAgent(None, None, registry=StubRegistry(names=names), menu_seed=seed))
+    assert order(1) == order(1) and sorted(order(1)) == names
+    assert len({tuple(order(s)) for s in range(6)}) > 1  # different sessions see different orders
+    a = SteeringAgent(None, None, registry=StubRegistry(names=names))
+    assert isinstance(a.menu_seed, int) and _menu(a) == _menu(a)  # stable within a session
+
+
+def test_vectors_created_mid_session_join_the_end_of_the_menu(creator):
+    before = _menu(creator)
+    creator._execute(ToolCall("create", {"concept": "wonder", "more": "x", "less": "y"}, ""))
+    assert _menu(creator) == before + ["wonder"]

tests/test_steering_gpu.py

@@ -97,7 +97,8 @@ def test_steer_tool_call_takes_effect_next_turn(agent):
 
 def _greedy_with(control, strength, max_new_tokens=40):
     a = SteeringAgent(_MODEL["model"], _MODEL["tokenizer"], registry=_MODEL["agent"].registry,
-                      model_name=config.MODEL_NAME, control=control, control_seed=1)
+                      model_name=config.MODEL_NAME, control=control, control_seed=1,
+                      menu_seed=_MODEL["agent"].menu_seed)  # same prompt as the fixture agent
     if strength:
         assert a.apply(VECTOR, strength)[0]
     inputs = _inputs(a)