Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changes/551.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
**Chats and apps with memory off no longer learn what is in your memory from the "didn't save" note.** When a reply
claimed a memory save that didn't happen, Sentient left the claim alone if a saved memory matched it. In a turn without
memory, which claims got the note showed what was saved. Now every such claim there gets the note.
9 changes: 5 additions & 4 deletions sentient/agent/loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -2201,7 +2201,7 @@ async def persist(role_: str, content: str | None, **fields: Any) -> None:
leftover = release()

try: # the turn counts as busy until here, also when the caller stops reading at Done
if correction := await self._memory_correction(result, incognito):
if correction := await self._memory_correction(result, incognito, memory_allowed):
result.text += correction
yield TextDelta(text=correction, **ev)
if result.error and not result.text:
Expand Down Expand Up @@ -2500,14 +2500,15 @@ async def trust_chat(self, session_id: str, *, via: str = "desktop", upto: str |
self._publish_mark(session_id, "")
return TrustResult(was=was, trusted=True)

async def _memory_correction(self, result: LoopResult, incognito: bool) -> str:
async def _memory_correction(self, result: LoopResult, incognito: bool, memory_allowed: bool) -> str:
"""The line added to a chat reply that says it saved a memory when no memory_remember call worked this turn
(#526), or "". A claim that a saved memory already backs (an earlier turn's save, or one made from the Memory
screen) is left alone: "Yes, I saved that you wake up at 7" can be about a save before this turn."""
screen) is left alone: "Yes, I saved that you wake up at 7" can be about a save before this turn. A turn
without memory never looks: which claims it corrects would tell what is stored (#551)."""
if result.error or not result.text:
return ""
claim = outcomes.unmade_memory_claim(result.text, result.memory_saves)
if not claim or await self._memory_backs(claim):
if not claim or (memory_allowed and await self._memory_backs(claim)):
return ""
note = outcomes.MEMORY_NOT_SAVED_INCOGNITO if incognito else outcomes.MEMORY_NOT_SAVED
if f"Correction: {note}" in result.text: # copied from an earlier correction in the chat: one is enough
Expand Down
43 changes: 43 additions & 0 deletions tests/test_memory_claims.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,49 @@ async def test_an_incognito_chat_says_it_keeps_nothing(config, isolated_home):
await s.stop()


async def test_a_chat_without_memory_corrects_a_claim_whatever_is_stored(config, isolated_home):
"""Without memory a stored memory never decides the correction, or it would tell what is stored (#551)."""
config.chat.incognito_uses_memory = False
kulkarni, mehta = "I've saved that your dentist is Dr. Kulkarni.", "I've saved that your dentist is Dr. Mehta."
llm = FakeProvider(replies=[kulkarni, kulkarni, mehta])
s = await _app(config, isolated_home, llm, "memory-off")
try:
await s.memory.remember("Maya's dentist is Dr. Kulkarni", use_llm=False)
text, _ = await _turn(s, await s.store.create_session(channel="cli"), "who is my dentist?")
assert text == kulkarni # with memory, the stored memory backs the claim as before
for reply in (kulkarni, mehta):
sid = await s.store.create_session(channel="cli", incognito=True)
text, _ = await _turn(s, sid, "who is my dentist?")
assert text == reply + "\n\nCorrection: " + outcomes.MEMORY_NOT_SAVED_INCOGNITO
finally:
await s.stop()


async def test_a_turn_without_memory_never_looks_up_stored_memory(config, isolated_home, monkeypatch):
"""The same holds when ``Agent.memory_allowed`` is False for another reason, as for /v1 with allow_memory off (#551)."""
kulkarni, mehta = "I've saved that your dentist is Dr. Kulkarni.", "I've saved that your dentist is Dr. Mehta."
llm = FakeProvider(replies=[kulkarni, mehta])
s = await _app(config, isolated_home, llm, "memory-off-lookup")
try:
await s.memory.remember("Maya's dentist is Dr. Kulkarni", use_llm=False)
looked_up = []
backs = s.agent._memory_backs

async def spy(claim):
looked_up.append(claim)
return await backs(claim)

monkeypatch.setattr(s.agent, "_memory_backs", spy)
monkeypatch.setattr(s.agent, "memory_allowed", lambda session, channel: False)
for reply in (kulkarni, mehta):
sid = await s.store.create_session(channel="cli")
text, _ = await _turn(s, sid, "who is my dentist?")
assert text == reply + "\n\nCorrection: " + outcomes.MEMORY_NOT_SAVED
assert looked_up == []
finally:
await s.stop()


def test_memory_claims_are_recognized_narrowly():
claims = [
"I've saved that you wake up at 7 a.m.",
Expand Down
Loading