diff --git a/src/toad/markdown_preparation.py b/src/toad/markdown_preparation.py index ec8061cb5..8325798b1 100644 --- a/src/toad/markdown_preparation.py +++ b/src/toad/markdown_preparation.py @@ -1,7 +1,11 @@ """Process-safe conversation parsing and code-fence highlighting data.""" +from copy import deepcopy +from collections.abc import Iterable, Iterator from dataclasses import dataclass, field from functools import cached_property +from hashlib import sha256 +import pickle from markdown_it.token import Token from textual.content import Content @@ -13,9 +17,82 @@ @dataclass(frozen=True) class PreparedMarkdownPart: - """One bounded native source part, without an invented transcript role.""" - - text: str + """Source and syntax acquired through the conversation parser declaration. + + Syntax precedes project-link resolution. Every document acquires its own + mutable tokens; neither partitioning nor presentation mutates this source. + Custom parser grammars retain their native acquisition instead. + """ + + text: str = field(compare=False) + tokens: tuple[Token, ...] = field(compare=False, repr=False) + revision: bytes = field(init=False) + + def __post_init__(self) -> None: + # Source acquisition/partitioning runs in preparation. Native refresh + # compares this revision without walking an arbitrarily nested tree. + object.__setattr__(self, "revision", sha256(pickle.dumps((self.text, self.tokens), protocol=5)).digest()) + + @classmethod + def capture(cls, text: str) -> "PreparedMarkdownPart": + from toad.conversation_markdown import parse_markdown_syntax + + return cls(text, tuple(parse_markdown_syntax(text))) + + def acquire_tokens(self) -> list[Token]: + return deepcopy(list(self.tokens)) + + def select_blocks(self, text: str, first: int, tokens: slice) -> "PreparedMarkdownPart": + """Select complete top-level blocks, retaining their closing tokens.""" + if text == self.text and first == 0 and tokens.start == 0 and tokens.stop == len(self.tokens): + return self + selected = deepcopy(list(self.tokens[tokens])) + for token in self._descendants(selected): + if token.map is not None: + token.map = [line - first for line in token.map] + return PreparedMarkdownPart(text, tuple(selected)) + + def select_table_rows(self, ranges: Iterable[tuple[int, int]]) -> Iterator["PreparedMarkdownPart"]: + """Repeat the original parsed header, never parse a synthetic table.""" + lines = self.text.splitlines(keepends=True) + rows = {} + body_first = body_stop = 0 + for index, token in enumerate(self.tokens): + if token.type == "tbody_open": + body_first = index + 1 + elif token.type == "tbody_close": + body_stop = index + break + elif body_first and token.type == "tr_open": + assert token.map is not None + rows[token.map[0]] = index + table = self.tokens[0] + assert table.type == "table_open" and table.map is not None + table_stop = table.map[1] + for first, stop in ranges: + row_first = rows.get(first, body_stop) + row_stop = rows.get(stop, body_stop) + tokens = deepcopy(list((*self.tokens[:body_first], + *self.tokens[row_first:row_stop], + *self.tokens[body_stop:]))) + length = 2 + min(stop, table_stop) - first + for token in self._descendants(tokens): + if token.map is None: + continue + if token.type == "table_open": + token.map = [0, length] + elif token.type == "tbody_open": + token.map = [2, length] + elif token.map[0] >= 2: + token.map = [line - first + 2 for line in token.map] + yield PreparedMarkdownPart("".join((*lines[:2], *lines[first:stop])), tuple(tokens)) + + @staticmethod + def _descendants(tokens): + for token in tokens: + yield token + if token.children is not None: + yield from PreparedMarkdownPart._descendants(token.children) @cached_property def retained_bytes(self) -> int: @@ -33,24 +110,20 @@ class PreparedFence: class PreparedMarkdown: tokens: list[Token] fences: dict[FenceKey, PreparedFence] - inlines: dict[int, Content] = field(default_factory=dict) def acquire_inline_content(self, tokens: list[Token]) -> "PreparedMarkdown": """Bind native content to the actual delivered, link-resolved tokens. - Token identities belong to this one acquisition, not the reusable - parser result or another document. Native block construction consumes - these same token objects and retains the resulting Content itself. + The parser's token metadata owns its acquired content. Native block + construction consumes these same tokens, without an identity registry. """ - def descendants(source): - for token in source: - yield token - if token.children is not None: - yield from descendants(token.children) - - inlines = {id(token): MarkdownBlock._token_to_content(token) - for token in descendants(tokens) if token.type == "inline"} - return PreparedMarkdown(tokens, self.fences, inlines) + for token in PreparedMarkdownPart._descendants(tokens): + if token.type == "inline": + token.meta[PreparedMarkdown] = MarkdownBlock._token_to_content(token) + return PreparedMarkdown(tokens, self.fences) + + def inline_content(self, token: Token) -> Content: + return token.meta[PreparedMarkdown] def prepare_tokens(tokens: list[Token], ansi: bool, dark: bool) -> PreparedMarkdown: diff --git a/src/toad/render_tasks.py b/src/toad/render_tasks.py index 15b4e5937..fe9b13391 100644 --- a/src/toad/render_tasks.py +++ b/src/toad/render_tasks.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from toad.widgets.patch_diff import PreparedPatch + from toad.work_preparation import PreparationRuntime, RenderPreparation, WorkKey @dataclass(frozen=True) @@ -59,16 +60,25 @@ def accept_result(self, result: object) -> PreparedPatch: @dataclass(frozen=True) class MarkdownRenderTask(ReusableRenderTask[PreparedMarkdown]): - """Parse and highlight one detached body before its independent delivery.""" + """Acquire syntax once, then highlight before independent delivery.""" - source: str + source: str | PreparedMarkdownPart ansi: bool dark: bool - def execute(self) -> PreparedMarkdown: - from toad.conversation_markdown import parse_markdown_syntax + async def preparation_identity(self, work: RenderPreparation, runtime: PreparationRuntime) -> WorkKey: + if isinstance(self.source, PreparedMarkdownPart): + from toad.work_preparation import WorkKey - return prepare_tokens(parse_markdown_syntax(self.source), self.ansi, self.dark) + # Acquired syntax owns its revision; accounting and delivery copies + # cannot change the original preparation identity. + return WorkKey(type(work), (type(self), self.source.revision, self.ansi, self.dark), work.scope) + return await super().preparation_identity(work, runtime) + + def execute(self) -> PreparedMarkdown: + part = (PreparedMarkdownPart.capture(self.source) + if isinstance(self.source, str) else self.source) + return prepare_tokens(part.acquire_tokens(), self.ansi, self.dark) def accept_result(self, result: object) -> PreparedMarkdown: if not isinstance(result, PreparedMarkdown): @@ -80,12 +90,21 @@ def accept_result(self, result: object) -> PreparedMarkdown: class MarkdownPartsTask(ReusableRenderTask[tuple[PreparedMarkdownPart, ...]]): """Partition one original message with the shared Markdown block budget.""" - source: str + source: str | PreparedMarkdownPart + + async def preparation_identity(self, work: RenderPreparation, runtime: PreparationRuntime) -> WorkKey: + if isinstance(self.source, PreparedMarkdownPart): + from toad.work_preparation import WorkKey + + return WorkKey(type(work), (type(self), self.source.revision), work.scope) + return await super().preparation_identity(work, runtime) def execute(self) -> tuple[PreparedMarkdownPart, ...]: from toad.widgets.transcript_fragments import RenderBudget - parts = tuple(PreparedMarkdownPart(text) for text in RenderBudget().split(self.source)) + part = (PreparedMarkdownPart.capture(self.source) + if isinstance(self.source, str) else self.source) + parts = tuple(RenderBudget().split(part)) for part in parts: part.retained_bytes return parts diff --git a/src/toad/widgets/agent_response.py b/src/toad/widgets/agent_response.py index 5398b742e..6e92fbc8f 100644 --- a/src/toad/widgets/agent_response.py +++ b/src/toad/widgets/agent_response.py @@ -8,6 +8,7 @@ from toad.widgets.message_divider import MessageClock, LiveMessageClock, MessageDivider from toad.widgets.route_header import RouteHeader from toad import response_delivery +from toad.markdown_preparation import PreparedMarkdownPart from toad.widgets.message_filter import ( CategorizedBlock, MessageCategory, @@ -32,10 +33,12 @@ def message_category(self) -> type[MessageCategory]: def __init__(self, markdown: str | None = None, *, delivery: response_delivery.ResponseDelivery = response_delivery.UnroutedResponse(), category: type[MessageCategory] | None = None, + markdown_part: PreparedMarkdownPart | None = None, paginate: bool = True, show_divider: bool = True, clock: MessageClock = LiveMessageClock()) -> None: self._message_category = category or delivery.category super().__init__( markdown, + markdown_part=markdown_part, paginate=paginate, **self.dispatch_sync(delivery, clock, show_divider), ) diff --git a/src/toad/widgets/incoming_message.py b/src/toad/widgets/incoming_message.py index 91c24e504..e5d426c7a 100644 --- a/src/toad/widgets/incoming_message.py +++ b/src/toad/widgets/incoming_message.py @@ -12,6 +12,7 @@ from toad.widgets.committed_presentation import CommitParticipant, SequenceClaim from toad.widgets.wire_message_handling import WireMessageHandling from toad.widgets.message_notifications import MessageNotifications +from toad.markdown_preparation import PreparedMarkdownPart @@ -26,10 +27,12 @@ def message_category(self) -> type[MessageCategory]: def commit_claim(self) -> SequenceClaim: return SequenceClaim(self.sequence) - def __init__(self, event: IncomingTranscript, *, show_header: bool = True) -> None: + def __init__(self, event: IncomingTranscript, *, show_header: bool = True, + markdown_part: PreparedMarkdownPart | None = None) -> None: super().__init__() self.event = event self.show_header = show_header + self.markdown_part = markdown_part @property def message_reference(self): @@ -46,7 +49,8 @@ def compose(self) -> ComposeResult: yield MessageDivider(f"Inbound · @{self.event.route.sender}", clock=MessageClock.recorded(self.event.timestamp)) yield RouteHeader(self.event.route, incoming=True) - yield AgentResponse(self.event.text, show_divider=False).add_class("routed-body") + yield AgentResponse(self.event.text, show_divider=False, + markdown_part=self.markdown_part).add_class("routed-body") if self.handling_references: yield MessageNotifications() diff --git a/src/toad/widgets/outgoing_message.py b/src/toad/widgets/outgoing_message.py index 2a99492d3..a8d07af68 100644 --- a/src/toad/widgets/outgoing_message.py +++ b/src/toad/widgets/outgoing_message.py @@ -11,15 +11,18 @@ from toad.widgets.message_filter import CategorizedBlock, OutboundCategory from toad.widgets.message_notifications import MessageNotifications from toad.widgets.wire_message_handling import WireMessageHandling +from toad.markdown_preparation import PreparedMarkdownPart class OutgoingMessage(WireMessageHandling, ConversationBlock, CategorizedBlock, VerticalGroup): DEFAULT_CLASSES = "block" - def __init__(self, event: SentTranscript, *, show_header: bool = True): + def __init__(self, event: SentTranscript, *, show_header: bool = True, + markdown_part: PreparedMarkdownPart | None = None): super().__init__() self.event = event self.show_header = show_header + self.markdown_part = markdown_part @property def message_reference(self): @@ -31,6 +34,7 @@ def message_category(self): def compose(self) -> ComposeResult: yield AgentResponse(self.event.text, + markdown_part=self.markdown_part, delivery=ResponseDelivery.from_route(self.event.routing.reply), show_divider=self.show_header, clock=MessageClock.recorded(self.event.timestamp)) diff --git a/src/toad/widgets/prepared_markdown.py b/src/toad/widgets/prepared_markdown.py index 974f4c284..ed918074d 100644 --- a/src/toad/widgets/prepared_markdown.py +++ b/src/toad/widgets/prepared_markdown.py @@ -20,10 +20,9 @@ from toad.app import ToadApp from toad.conversation_markdown import ConversationCodeFence, ConversationMarkdown, _ThreadLocalPathParser -from toad.markdown_preparation import PreparedMarkdown +from toad.markdown_preparation import PreparedMarkdown, PreparedMarkdownPart from toad.block_content import MarkdownBlockContent from toad.render_tasks import MarkdownRenderTask -from toad.widgets.transcript_fragments import RenderBudget from toad.widgets.viewport_body import MeasuredViewportBody from toad.widgets.worker_static import WorkerStatic @@ -227,12 +226,32 @@ def __init__( self, markdown: str | None = None, *, name: str | None = None, id: str | None = None, classes: str | None = None, parser_factory: Callable[[], MarkdownIt] | None = None, open_links: bool = False, + markdown_part: PreparedMarkdownPart | None = None, ) -> None: self._prepared_markdown: PreparedMarkdown | None = None + self._markdown_part = markdown_part factory = self._make_parser if parser_factory is None else parser_factory super().__init__(markdown, name=name, id=id, classes=classes, parser_factory=factory, open_links=open_links) + @property + def partitionable_syntax(self) -> bool: + """Only the original conversation factory declares this grammar. + + An overridden factory or arbitrary custom parser has no declaration + permitting conversation-syntax partitioning, even if today's output + happens to look the same. + """ + return getattr(self._parser_factory, "__func__", None) is ConversationMarkdown._make_parser + + def acquired_source(self, markdown: str) -> str | PreparedMarkdownPart: + part = self._markdown_part + return part if part is not None and part.text == markdown else markdown + + def update_part(self, part: PreparedMarkdownPart) -> AwaitComplete: + self._markdown_part = part + return self.update(part.text) + def reconstructible_children(self) -> tuple[Widget, ...]: """Native resources rebuilt from this document's original source.""" return tuple(child for child in self.children if isinstance(child, MarkdownBlock)) @@ -259,9 +278,13 @@ def append(self, markdown: str) -> AwaitComplete: return self.publish_body(partial(self._append_body_source, markdown)) def _update_body_source(self, markdown: str) -> AwaitComplete: + if self._markdown_part is not None and self._markdown_part.text != markdown: + self._markdown_part = None return super().update(markdown) def _append_body_source(self, markdown: str) -> AwaitComplete: + if self._markdown_part is not None and self._markdown_part.text != self.source + markdown: + self._markdown_part = None return super().append(markdown) def on_unmount(self) -> None: @@ -273,7 +296,7 @@ async def _parse_tokens( # Custom parser factories retain their native semantics. This process # function implements the declaration-owned conversation/path parser. parent = self.parent - if not isinstance(parser, _ThreadLocalPathParser): + if type(parser) is not _ThreadLocalPathParser: tokens = await super()._parse_tokens(parser, markdown, use_thread=use_thread) if tokens is not None: prepared = await asyncio.to_thread( @@ -284,22 +307,24 @@ async def _parse_tokens( self._prepared_markdown = prepared return tokens if not isinstance(self.app, ToadApp): - tokens = (await asyncio.to_thread(parser.parse, markdown) - if use_thread else parser.parse(markdown)) - prepared = await asyncio.to_thread( - PreparedMarkdown(tokens, {}).acquire_inline_content, tokens, - ) + source = self.acquired_source(markdown) + def acquire_local(): + tokens = (parser.resolve_tokens(source.acquire_tokens()) + if isinstance(source, PreparedMarkdownPart) else parser.parse(source)) + return PreparedMarkdown(tokens, {}).acquire_inline_content(tokens) + + prepared = await asyncio.to_thread(acquire_local) if self._closing or self._pruning or self.parent is not parent: return None self._prepared_markdown = prepared - return tokens + return prepared.tokens # Runway and delivery use the same pure preparation key. Filesystem # links are resolved only in each independent delivered token resource, # after highlighting; they never become reusable renderer inputs. while not self._closing and self.is_attached and not self._pruning: theme = (self.app.native_ansi_color, self.app.current_theme.dark) - request = self.app.render_processes.submit(MarkdownRenderTask(markdown, *theme)) + request = self.app.render_processes.submit(MarkdownRenderTask(self.acquired_source(markdown), *theme)) worker = self.run_worker(request, group="markdown-preparation", exit_on_error=False) try: prepared = await worker.wait() @@ -338,7 +363,7 @@ def _get_token_content(self, token: Token, *, block: MarkdownBlock) -> Content: # Its declared answer cannot be replaced by the native pure result. return super()._get_token_content(token, block=block) assert self._prepared_markdown is not None - return self._prepared_markdown.inlines[id(token)] + return self._prepared_markdown.inline_content(token) def get_block_class(self, block_name: str) -> type[MarkdownBlock]: if block_name in {"fence", "code_block"}: diff --git a/src/toad/widgets/streaming_markdown.py b/src/toad/widgets/streaming_markdown.py index 6f518cbbe..dd8b0aed6 100644 --- a/src/toad/widgets/streaming_markdown.py +++ b/src/toad/widgets/streaming_markdown.py @@ -42,7 +42,8 @@ def compose(self) -> ComposeResult: yield from self._prefix def _body(self, fragment: PreparedMarkdownPart) -> PreparedConversationMarkdown: - return PreparedConversationMarkdown(fragment.text, classes="-message-fragment") + return PreparedConversationMarkdown(fragment.text, markdown_part=fragment, + classes="-message-fragment") @property def retained_source_bytes(self) -> int: @@ -60,7 +61,7 @@ async def _prepare_parts(self, parts, current: Callable[[], bool]) -> bool: if not current(): return False await asyncio.gather(*(self.app.render_processes.prepare(MarkdownRenderTask( - part.text, self.app.native_ansi_color, self.app.current_theme.dark, + part, self.app.native_ansi_color, self.app.current_theme.dark, )) for part in parts[first:first + self.batch_size])) return current() @@ -127,18 +128,30 @@ async def _publish_content(self, source: str, text: str, append: bool, async with self._content_lock: if not is_current(): return - if self.body_dormant: + # MaterializingBody is dormant while publishing, even when its + # previous native roots remain usable. Append needs reconstruction + # only after those roots have actually been retired. + if not super().reconstructible_children(): self._needs_full_markdown_update = True - fragments = (await self.app.render_processes.submit(MarkdownPartsTask(source)) - if self._paginate and not RenderBudget().fits(source) else ()) + if self._markdown_part is not None and self._markdown_part.text != source: + self._markdown_part = None + fragments = (await self.app.render_processes.submit(MarkdownPartsTask(self.acquired_source(source))) + if self._paginate and self.partitionable_syntax + and not RenderBudget().fits(source) else ()) if not is_current(): return if len(fragments) <= 1: + if fragments: + self._markdown_part = fragments[0] if self.fragments: await self.remove_children(self.fragment_views) self.fragments, self._fragment_views = (), () self.start = self.stop = 0 - if append and self.source + text == source and not self._needs_full_markdown_update: + # Paged views replaced the native roots. Reconstruct the + # body before native append can update its final block. + self._needs_full_markdown_update = True + if (append and self.source + text == source + and not self._needs_full_markdown_update): await self._append_body_source(text) else: await self._update_body_source(source) @@ -161,8 +174,9 @@ async def _publish_content(self, source: str, text: str, append: bool, for index, child in previous.items(): if not is_current(): return - if selected.start <= index < selected.stop and child.source != fragments[index].text: - await child.update(fragments[index].text) + if (selected.start <= index < selected.stop + and child._markdown_part != fragments[index]): + await child.update_part(fragments[index]) async with window.preserve_history(None, root=self): if not is_current(): return diff --git a/src/toad/widgets/transcript_fragments.py b/src/toad/widgets/transcript_fragments.py index a5b9d975b..21718b2f6 100644 --- a/src/toad/widgets/transcript_fragments.py +++ b/src/toad/widgets/transcript_fragments.py @@ -1,19 +1,18 @@ """Render-sized transcript fragments; wire records and cursors remain model-owned.""" import asyncio -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from functools import cached_property from collections.abc import Iterator from typing import TYPE_CHECKING -from threading import local from agent_comms.mro_dispatch import MroDispatch, handles from agent_comms.transcript_events import ( TranscriptEvent, TextTranscript, ContextTranscript, UserTranscript, MarkdownTranscript, ToolTranscript, ) -from markdown_it import MarkdownIt from toad.render_backend import ReusableRenderTask +from toad.markdown_preparation import PreparedMarkdownPart from toad.acp.status import ToolCallStatus from toad.widgets.agent_activity import AgentActivityBoundary from toad.widgets.message_filter import event_category, keep_events @@ -22,19 +21,6 @@ from toad.render_backend import Renderer from toad.tool_output import ToolOutputPart -class _FragmentParserState(local): - def __init__(self) -> None: - self.parser = MarkdownIt("gfm-like") - - -_parser_state = _FragmentParserState() - - -def _fragment_parser() -> MarkdownIt: - """Reuse parser machinery within one execution thread, never concurrently.""" - return _parser_state.parser - - async def prepare_transcript_fragments( events: tuple[TranscriptEvent, ...], pool: "Renderer | None" = None, *, continuation: bool = False, @@ -63,38 +49,39 @@ class RenderBudget: def fits(self, text: str) -> bool: return len(text) <= self.characters and len(text.splitlines()) <= self.lines - def _split_table(self, text: str) -> Iterator[str]: + def _split_table(self, part: PreparedMarkdownPart) -> Iterator[PreparedMarkdownPart]: """Page a large GFM table by rows, repeating its required header.""" - lines = text.splitlines(keepends=True) + lines = part.text.splitlines(keepends=True) if len(lines) <= 2: - yield text + yield part return header = "".join(lines[:2]) - rows: list[str] = [] + first = 2 row_characters = 0 row_limit = max(1, self.lines - 2) character_limit = max(self.characters, len(header) + 1) - for row in lines[2:]: - if rows and ( - len(rows) >= row_limit + table = part.tokens[0] + assert table.map is not None + ranges = [] + for index, row in enumerate(lines[2:table.map[1]], 2): + if index > first and ( + index - first >= row_limit or len(header) + row_characters + len(row) > character_limit ): - yield header + "".join(rows) - rows = [] + ranges.append((first, index)) + first = index row_characters = 0 - rows.append(row) row_characters += len(row) - if rows: - yield header + "".join(rows) + if first < len(lines): + ranges.append((first, len(lines))) + yield from part.select_table_rows(ranges) - def split(self, text: str) -> Iterator[str]: + def split(self, source: PreparedMarkdownPart) -> Iterator[PreparedMarkdownPart]: + text = source.text if not text: return if self.fits(text): - # No block can exceed either budget when the entire document fits. - # Return the unchanged source; there is no partition decision to - # parse, and normal Markdown rendering still handles its syntax. - yield text + yield source return lines = text.splitlines(keepends=True) offsets = [0] @@ -105,43 +92,58 @@ def split(self, text: str) -> Iterator[str]: # between top-level Markdown blocks so rendering never invents a line # break at an arbitrary provider-chunk or character boundary. blocks = { - token.map[0]: token.type - for token in _fragment_parser().parse(text) + token.map[0]: (index, token.type) + for index, token in enumerate(source.tokens) if token.level == 0 and token.map is not None } starts = sorted({0, *blocks, len(lines)}) - parts: list[tuple[str, str | None]] = [ - (text[offsets[start] : offsets[stop]], blocks.get(start)) + parts = [ + (start, stop, blocks.get(start, (0, None))[0], + blocks.get(stop, (len(source.tokens), None))[0], blocks.get(start, (0, None))[1]) for start, stop in zip(starts, starts[1:]) ] - pending = "" + pending_first = 0 + pending_stop = 0 + pending_token_first = 0 + pending_token_stop = 0 + pending_characters = 0 pending_lines = 0 - for part, kind in parts: + def select(first, stop, token_first, token_stop): + return source.select_blocks(text[offsets[first]:offsets[stop]], first, + slice(token_first, token_stop)) + + for start, stop, token_first, token_stop, kind in parts: + part = text[offsets[start]:offsets[stop]] if kind == "table_open" and ( len(part) > self.characters or part.count("\n") > self.lines ): - if pending: - yield pending - pending = "" + if pending_characters: + yield select(pending_first, pending_stop, pending_token_first, pending_token_stop) + pending_characters = 0 pending_lines = 0 - yield from self._split_table(part) + yield from self._split_table(select(start, stop, token_first, token_stop)) continue part_lines = part.count("\n") + int(bool(part) and not part.endswith("\n")) exceeds = ( - pending + pending_characters and ( - len(pending) + len(part) > self.characters + pending_characters + len(part) > self.characters or pending_lines + part_lines > self.lines ) ) if exceeds: - yield pending - pending = "" + yield select(pending_first, pending_stop, pending_token_first, pending_token_stop) + pending_characters = 0 pending_lines = 0 - pending += part + if not pending_characters: + pending_first = start + pending_token_first = token_first + pending_stop = stop + pending_token_stop = token_stop + pending_characters += len(part) pending_lines += part_lines - if pending: - yield pending + if pending_characters: + yield select(pending_first, pending_stop, pending_token_first, pending_token_stop) @dataclass(frozen=True) @@ -149,11 +151,13 @@ class TranscriptFragment: events: tuple[TranscriptEvent, ...] continuation: bool = False starts_agent_activity: bool = False + markdown_part: PreparedMarkdownPart | None = field(default=None, kw_only=True) def blocks(self, *, fragment: bool = False, show_divider: bool = True): from toad.widgets.transcript_history import TranscriptBlockConsumer - consumer = TranscriptBlockConsumer(fragment=fragment, show_divider=show_divider) + consumer = TranscriptBlockConsumer(fragment=fragment, show_divider=show_divider, + markdown_part=self.markdown_part) for event in self.events: consumer.dispatch_sync(event) return consumer.blocks @@ -198,10 +202,11 @@ def context(self, event: ContextTranscript): @handles(UserTranscript, MarkdownTranscript) def text(self, event: TextTranscript): starts_activity = self.boundary.observe(event_category(event)) if event.starts_activity else False + source = PreparedMarkdownPart.capture(event.text) self.fragments.extend( - TranscriptFragment((replace(event, text=part),), continuation=self.continuation or index > 0, - starts_agent_activity=starts_activity and index == 0) - for index, part in enumerate(self.budget.split(event.text) if self.split_text else (event.text,)) + TranscriptFragment((replace(event, text=part.text),), continuation=self.continuation or index > 0, + starts_agent_activity=starts_activity and index == 0, markdown_part=part) + for index, part in enumerate(self.budget.split(source) if self.split_text else (source,)) ) @handles(ToolTranscript) @@ -259,22 +264,23 @@ async def prepare_fragments(self, fragments, keep_going, *, batch_size: int) -> for first in range(0, len(fragments), batch_size): if not keep_going(): return - await asyncio.gather(*(self.dispatch(event) + await asyncio.gather(*(self.dispatch(event, fragment.markdown_part) for fragment in fragments[first:first + batch_size] if self.selected is None or keep_events(fragment.events, self.selected) for event in fragment.events)) @handles(TranscriptEvent) - async def undisclosed(self, event: TranscriptEvent) -> None: + async def undisclosed(self, event: TranscriptEvent, part: PreparedMarkdownPart | None) -> None: # Metadata and tool disclosure contents retain their existing lazy # owners. A viewport prediction does not open those disclosures. pass - @handles(MarkdownTranscript) - async def markdown(self, event: MarkdownTranscript) -> None: + @handles(UserTranscript, MarkdownTranscript) + async def markdown(self, event: TextTranscript, part: PreparedMarkdownPart | None) -> None: from toad.render_tasks import MarkdownRenderTask - await self.renderer.prepare(MarkdownRenderTask(event.text, self.ansi, self.dark)) + await self.renderer.prepare(MarkdownRenderTask(part if part is not None else event.text, + self.ansi, self.dark)) @dataclass(frozen=True) diff --git a/src/toad/widgets/transcript_history.py b/src/toad/widgets/transcript_history.py index 9d32c9424..16f52783f 100644 --- a/src/toad/widgets/transcript_history.py +++ b/src/toad/widgets/transcript_history.py @@ -37,6 +37,7 @@ ) from toad.response_delivery import ResponseDelivery from toad.widgets.prepared_markdown import PreparedContentRange +from toad.markdown_preparation import PreparedMarkdownPart from toad.widgets.agent_response import AgentResponse from toad.widgets.agent_thought import AgentThought from toad.widgets.tool_call import ToolCall @@ -60,10 +61,12 @@ class _PublicationRetired(Exception): """Unwind an anchor transaction whose source owner no longer publishes.""" class TranscriptBlockConsumer(MroDispatch): - def __init__(self, *, fragment: bool, show_divider: bool): + def __init__(self, *, fragment: bool, show_divider: bool, + markdown_part: PreparedMarkdownPart | None = None): self.blocks: list[Widget] = [] self.fragment = fragment self.show_divider = show_divider + self.markdown_part = markdown_part @handles(ContextTranscript) def context(self, event: ContextTranscript): @@ -73,30 +76,35 @@ def context(self, event: ContextTranscript): @handles(UserTranscript) def user(self, event: UserTranscript): self.blocks.append(UserInput(event.text, claim=TranscriptInputClaim(event), + markdown_part=self.markdown_part, show_divider=self.show_divider, clock=MessageClock.recorded(event.timestamp))) @handles(IncomingTranscript) def incoming(self, event: IncomingTranscript): from toad.widgets.incoming_message import IncomingMessage - self.blocks.append(IncomingMessage(event, show_header=self.show_divider)) + self.blocks.append(IncomingMessage(event, show_header=self.show_divider, + markdown_part=self.markdown_part)) @handles(SentTranscript) def sent(self, event: SentTranscript): from toad.widgets.outgoing_message import OutgoingMessage - self.blocks.append(OutgoingMessage(event, show_header=self.show_divider)) + self.blocks.append(OutgoingMessage(event, show_header=self.show_divider, + markdown_part=self.markdown_part)) @handles(AgentTextTranscript) def agent(self, event: AgentTextTranscript): self.blocks.append(AgentResponse( event.text, delivery=ResponseDelivery.from_route(event.routing.reply if event.routing else None), category=event_category(event), paginate=not self.fragment, + markdown_part=self.markdown_part, show_divider=self.show_divider, clock=MessageClock.recorded(event.timestamp), )) @handles(ThinkingTranscript) def thinking(self, event: ThinkingTranscript): - self.blocks.append(AgentThought(event.text, paginate=not self.fragment)) + self.blocks.append(AgentThought(event.text, paginate=not self.fragment, + markdown_part=self.markdown_part)) def transcript_blocks(events: tuple[TranscriptEvent, ...], *, fragment: bool = False, show_divider: bool = True) -> list[Widget]: diff --git a/src/toad/widgets/user_input.py b/src/toad/widgets/user_input.py index 6547f73ec..e1e168210 100644 --- a/src/toad/widgets/user_input.py +++ b/src/toad/widgets/user_input.py @@ -4,6 +4,7 @@ from textual.app import ComposeResult from textual import containers from toad.widgets.prepared_markdown import PreparedConversationMarkdown +from toad.markdown_preparation import PreparedMarkdownPart from toad.widgets.non_selectable_label import NonSelectableLabel from toad.widgets.message_divider import MessageDivider, MessageClock, LiveMessageClock @@ -44,9 +45,12 @@ class UserInput(ConversationBlock, SnapshotPresentation, CategorizedBlock, conta def message_category(self) -> type[MessageCategory]: return UserCategory - def __init__(self, content: str, *, claim: CommitClaim = CAPTURED_CLAIM, show_divider: bool = True, clock: MessageClock = LiveMessageClock()) -> None: + def __init__(self, content: str, *, claim: CommitClaim = CAPTURED_CLAIM, + markdown_part: PreparedMarkdownPart | None = None, + show_divider: bool = True, clock: MessageClock = LiveMessageClock()) -> None: super().__init__() self.content = content + self.markdown_part = markdown_part self.show_divider = show_divider self.clock = clock self.claim = claim @@ -60,7 +64,7 @@ def compose(self) -> ComposeResult: yield MessageDivider("User", clock=self.clock) with containers.HorizontalGroup(classes="user-input-body"): yield NonSelectableLabel("❯" if self.show_divider else " ", id="prompt") - yield PreparedConversationMarkdown(self.content, id="content") + yield PreparedConversationMarkdown(self.content, markdown_part=self.markdown_part, id="content") def get_clipboard_text(self) -> str | None: return self.content diff --git a/tests/test_worker_preparation_acquisition.py b/tests/test_worker_preparation_acquisition.py index d485e1cad..bec56c6c5 100644 --- a/tests/test_worker_preparation_acquisition.py +++ b/tests/test_worker_preparation_acquisition.py @@ -385,9 +385,12 @@ def input(): resources = tuple(markdown._prepared_markdown for markdown in body.query(PreparedConversationMarkdown) if markdown._prepared_markdown is not None) - assert resources and all(resource.inlines for resource in resources) + inline_contents = tuple(resource.inline_content(token) + for resource in resources for token in resource.tokens + if token.type == "inline") + assert resources and inline_contents assert any(span.style.meta.get("@click", "") - for resource in resources for content in resource.inlines.values() + for content in inline_contents for span in content.spans if not isinstance(span.style, str)) assert counts["ui_inline_conversions"] == 0 assert counts["worker_inline_conversions"] > 0 diff --git a/tests/viewport_body_lifetime_pilot.py b/tests/viewport_body_lifetime_pilot.py index fa23dfff2..341e0c680 100644 --- a/tests/viewport_body_lifetime_pilot.py +++ b/tests/viewport_body_lifetime_pilot.py @@ -230,9 +230,13 @@ async def main(): assert sum(doc.body_dormant for doc in docs) >= 20 assert all(doc.source == source(i) for i, doc in enumerate(docs)) from toad.widgets.viewport_body import MeasuredBody - cold = [doc for doc in docs if type(doc._body_measurement) is MeasuredBody] - assert cold, "Original widget budget never exercised cold eviction" - dormant = cold[0] + # Paint and native controls have separate byte and widget admission. + # Exercise cold reconstruction using the original paint release; + # this small source does not establish real byte-pressure eviction. + dormant = next(doc for doc in docs if doc.body_dormant + and not view.window.document_viewport.requires_body(doc)) + dormant.release_paint() + assert type(dormant._body_measurement) is MeasuredBody assert not dormant.query(MarkdownParagraph), "Cold body retained its native message pumps" before = len(app._registry) view.window.release_anchor()