diff --git a/api/mcp_server.py b/api/mcp_server.py index fa9127cf2..a59ff0092 100644 --- a/api/mcp_server.py +++ b/api/mcp_server.py @@ -19,6 +19,7 @@ """ import asyncio +import json import mimetypes import httpx from mcp.server import Server @@ -83,6 +84,86 @@ async def list_tools() -> list[Tool]: "required": ["job_id"], }, ), + Tool( + name="modly_inspect_mesh", + description=( + "Diagnose a mesh without modifying it: watertight, manifold, holes, " + "self-intersections, parts, dimensions, volume, UVs/normals, printability." + ), + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + }, + "required": ["path"], + }, + ), + Tool( + name="modly_suggest_fixes", + description="Inspect a mesh and list the operations (with params) that would fix its problems.", + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + }, + "required": ["path"], + }, + ), + Tool( + name="modly_repair_mesh", + description=( + "Repair a mesh automatically (holes, non-manifold edges, degenerate faces, " + "flipped normals) until it is closed and manifold. Writes a new, untextured mesh." + ), + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "fill_holes": { + "type": "boolean", + "description": "Close holes (default true).", + }, + }, + "required": ["path"], + }, + ), + Tool( + name="modly_arrange_mesh", + description=( + "Reposition a mesh in one call: auto_orient (lay flat), normalize_scale " + "(resize), recenter (centre on origin), drop_to_ground (rest on ground). " + "Steps always run in that order. Writes a new mesh file." + ), + inputSchema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh (e.g. 'Default/mesh.glb').", + }, + "steps": { + "type": "array", + "items": { + "type": "string", + "enum": ["auto_orient", "normalize_scale", "recenter", "drop_to_ground"], + }, + }, + "size": {"type": "number", "description": "normalize_scale: target size in `unit`."}, + "unit": {"type": "string", "enum": ["m", "cm", "mm"]}, + "dimension": {"type": "string", "enum": ["longest", "x", "y", "z"]}, + }, + "required": ["path", "steps"], + }, + ), Tool( name="modly_decimate_mesh", description="Reduce the polygon count of a mesh using quadric edge collapse decimation.", @@ -217,6 +298,34 @@ async def _dispatch(client: httpx.AsyncClient, name: str, args: dict) -> str: parts.append(f"Error: {s['error']}") return " | ".join(parts) + elif name in ("modly_inspect_mesh", "modly_suggest_fixes"): + op = name.removeprefix("modly_") + r = await client.post(f"{API_BASE}/optimize/op/{op}", json={"path": args["path"]}) + r.raise_for_status() + return json.dumps(r.json(), indent=2) + + elif name == "modly_repair_mesh": + params = {} if args.get("fill_holes") is None else {"fill_holes": bool(args["fill_holes"])} + r = await client.post( + f"{API_BASE}/optimize/op/repair_mesh", + json={"path": args["path"], "params": params}, + ) + r.raise_for_status() + return json.dumps(r.json(), indent=2) + + elif name == "modly_arrange_mesh": + scale = {k: args[k] for k in ("size", "unit", "dimension") if args.get(k) is not None} + r = await client.post( + f"{API_BASE}/optimize/arrange", + json={ + "path": args["path"], + "steps": args.get("steps") or [], + "params": {"normalize_scale": scale} if scale else {}, + }, + ) + r.raise_for_status() + return json.dumps(r.json(), indent=2) + elif name == "modly_decimate_mesh": r = await client.post( f"{API_BASE}/optimize/mesh", diff --git a/api/requirements.txt b/api/requirements.txt index 975f19ca3..0462a51ba 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -10,6 +10,9 @@ mcp>=1.0.0 # Mesh processing (optimize + export endpoints) trimesh>=4.5.0 pymeshlab>=2023.12 +# trimesh's graph engine: connected parts, hole loops and vertex merging in +# inspect_mesh / repair_mesh. Optional for trimesh, so never pulled in by it. +scipy>=1.10 # Model downloads (HuggingFace) huggingface_hub>=0.27.0 diff --git a/api/routers/agent.py b/api/routers/agent.py index 1fb389eb6..e2cb56efb 100644 --- a/api/routers/agent.py +++ b/api/routers/agent.py @@ -4,12 +4,12 @@ """ import asyncio import json -import re import uuid from typing import Optional import httpx from fastapi import APIRouter +from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from services import llm_server @@ -28,6 +28,10 @@ - **list_models** — List all downloaded 3D generation models ready to use. - **unload_models** — Unload all 3D generation models from GPU VRAM to free memory. - **get_mesh_info** — Get info about the current mesh in the 3D viewer (path, triangle count). +- **inspect_mesh(path)** — Diagnose a mesh without modifying it: watertight, manifold, holes, self-intersections, parts, dimensions, volume, UVs/normals, and whether it is 3D-printable. +- **suggest_fixes(path)** — Inspect a mesh and list the operations (with params) that would fix its problems. +- **repair_mesh(path, fill_holes?)** — Repair a mesh automatically: fills holes, fixes non-manifold edges, degenerate faces and flipped normals, re-checking until it is closed and manifold. Writes a new, untextured mesh. +- **arrange_mesh(path, steps, size?, unit?, dimension?)** — Reposition a mesh in one call. Steps: `auto_orient` (lay it flat on its widest side), `normalize_scale` (resize to `size` in `unit`), `recenter` (centre on the origin), `drop_to_ground` (rest it on the ground). Steps always run in that order, whatever order you list them in. - **decimate_mesh(path, target_faces)** — Reduce the polygon count of a mesh. - **smooth_mesh(path, iterations)** — Apply Laplacian smoothing to a mesh. - **get_generation_status(job_id)** — Poll the status of an ongoing 3D generation job. @@ -39,6 +43,9 @@ - Always use tools to act on the scene — never just describe what you would do. - If you need the current mesh path, call get_mesh_info first. +- To repair or fix a mesh, call repair_mesh. Report the issues it could not fix, and warn the user when it says the textures were dropped. +- To center, ground, lay flat or resize a mesh, call arrange_mesh ONCE with every requested step — never one call per step. +- To answer questions about a mesh's quality or printability, call inspect_mesh — never guess. Before repairing a mesh, call suggest_fixes to pick the right operations. - If you need to run a workflow but don't know the ID, call list_workflows first. - To create a workflow, ONLY use extension ids listed under "Available extensions" in the context. Never invent an id. Chain steps so each step's input type matches the previous step's output type. - For a workflow's input, `input_type` MUST be exactly one of: `image`, `text`, or `mesh`. These map to the Image, Text, and Load 3D Mesh nodes. Never invent another input. Pick the one matching the first step's expected input. @@ -72,6 +79,115 @@ "parameters": {"type": "object", "properties": {}}, }, }, + { + "type": "function", + "function": { + "name": "inspect_mesh", + "description": "Diagnose a mesh without modifying it: watertight, manifold, holes, self-intersections, disconnected parts, dimensions, volume, UVs/normals, and whether it is 3D-printable.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "suggest_fixes", + "description": "Inspect a mesh and list the operations (with parameters) that would fix its problems. Does not modify the mesh.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "repair_mesh", + "description": ( + "Repair a mesh automatically: fill holes, fix non-manifold edges, degenerate " + "faces and flipped normals, re-checking until it is closed and manifold. " + "Writes a new mesh without textures; does nothing if the mesh is already sound." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + "fill_holes": { + "type": "boolean", + "description": ( + "Close holes (default true). Set false only for meshes meant to stay open, " + "like terrain: then only flipped faces are fixed." + ), + }, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "arrange_mesh", + "description": ( + "Reposition a mesh in a single call: lay it flat, resize it, centre it and/or " + "rest it on the ground. Pass every requested step at once; they always run in " + "the order auto_orient → normalize_scale → recenter → drop_to_ground." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Workspace-relative path to the mesh file. Use get_mesh_info to obtain it.", + }, + "steps": { + "type": "array", + "items": { + "type": "string", + "enum": ["auto_orient", "normalize_scale", "recenter", "drop_to_ground"], + }, + "description": ( + "auto_orient: lay flat on its widest side (a standing figure is laid down). " + "normalize_scale: resize to `size`. recenter: centre on the origin. " + "drop_to_ground: lowest point at height 0." + ), + }, + "size": { + "type": "number", + "description": "normalize_scale only: target size of `dimension`, in `unit`.", + }, + "unit": { + "type": "string", + "enum": ["m", "cm", "mm"], + "description": "normalize_scale only: unit of `size`. Default m.", + }, + "dimension": { + "type": "string", + "enum": ["longest", "x", "y", "z"], + "description": "normalize_scale only: which side gets `size` (y is the height). Default longest.", + }, + }, + "required": ["path", "steps"], + }, + }, + }, { "type": "function", "function": { @@ -259,6 +375,64 @@ def _build_workflow_graph(name: str, description: str, input_type: str, steps: l return {"name": name, "description": description, "nodes": all_nodes, "edges": edges} +def _format_inspection(report: dict) -> str: + """Render an inspect_mesh report as compact text for the model.""" + def yes_no(value) -> str: + return "unknown" if value is None else ("yes" if value else "no") + + dims = " × ".join(f"{d:.4g}" for d in report.get("dimensions", [])) + volume = report.get("volume") + self_hits = report.get("self_intersecting_face_count") + lines = [ + f"Printable: {yes_no(report.get('printable'))}", + f"Faces: {report.get('face_count', '?'):,}, vertices: {report.get('vertex_count', '?'):,}, parts: {report.get('component_count', '?')}", + f"Watertight: {yes_no(report.get('is_watertight'))}, manifold: {yes_no(report.get('is_manifold'))}, consistent winding: {yes_no(report.get('is_winding_consistent'))}", + f"Holes: {report.get('hole_count', 0)} ({report.get('boundary_edge_count', 0)} open edges), non-manifold edges: {report.get('non_manifold_edge_count', 0)}, degenerate faces: {report.get('degenerate_face_count', 0)}", + f"Self-intersecting faces: {'not checked' if self_hits is None else self_hits}", + f"Dimensions: {dims}, surface area: {report.get('surface_area', 0):.4g}, volume: {'n/a (not closed)' if volume is None else f'{volume:.4g}'}", + f"UVs: {yes_no(report.get('has_uvs'))}, normals: {yes_no(report.get('has_normals'))}", + ] + issues = report.get("issues") or [] + if issues: + lines.append("Issues: " + "; ".join(issues)) + return "\n".join(lines) + + +def _format_fixes(data: dict) -> str: + """Render a suggest_fixes result as compact text for the model.""" + fixes = data.get("fixes") or [] + notes = data.get("notes") or [] + if not fixes and not notes: + return "No fixes needed — the mesh has no detected problems." + lines = ["Suggested operations:"] if fixes else ["No operation can fix the detected problems."] + for fix in fixes: + lines.append(f"- {fix['op']} {json.dumps(fix.get('params', {}))}: {fix.get('reason', '')}") + lines.extend(f"Note: {note}" for note in notes) + return "\n".join(lines) + + +def _format_repair(data: dict) -> tuple[str, dict | None]: + """Render a repair_mesh result for the model, plus the viewer update.""" + remaining = data.get("remaining_issues") or [] + if not data.get("repaired"): + text = "Nothing to repair: the mesh is already closed and manifold." + if remaining: + text = "Nothing was repaired. Remaining: " + "; ".join(remaining) + "." + return text, None + + steps = ", ".join(data.get("steps") or []) + lines = [ + f"Repaired in {data.get('passes', '?')} pass(es) ({steps}).", + ("Still remaining: " + "; ".join(remaining) + ".") if remaining + else "The mesh is now closed (watertight) and manifold.", + f"New mesh path: {data['path']} ({data.get('face_count', '?'):,} faces).", + ] + if data.get("textures_dropped"): + lines.append("The textures and UVs were dropped: the repaired mesh is geometry only.") + payload = {"type": "mesh_update", "url": data["url"], "face_count": data.get("face_count")} + return "\n".join(lines), payload + + async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, dict | None]: """Execute a tool and return (result_text, action_payload). action_payload carries data the frontend needs to react (e.g. new mesh URL). @@ -289,6 +463,55 @@ async def execute_tool(name: str, arguments: dict, context: dict) -> tuple[str, info += f" ({mesh_triangles:,} triangles)" return info, None + elif name == "inspect_mesh": + r = await client.post( + f"{MODLY_API}/optimize/op/inspect_mesh", + json={"path": arguments["path"]}, + ) + r.raise_for_status() + return _format_inspection(r.json()), None + + elif name == "suggest_fixes": + r = await client.post( + f"{MODLY_API}/optimize/op/suggest_fixes", + json={"path": arguments["path"]}, + ) + r.raise_for_status() + return _format_fixes(r.json()), None + + elif name == "repair_mesh": + params = {} + if arguments.get("fill_holes") is not None: + params["fill_holes"] = bool(arguments["fill_holes"]) + r = await client.post( + f"{MODLY_API}/optimize/op/repair_mesh", + json={"path": arguments["path"], "params": params}, + ) + r.raise_for_status() + return _format_repair(r.json()) + + elif name == "arrange_mesh": + scale_params = { + key: arguments[key] for key in ("size", "unit", "dimension") + if arguments.get(key) is not None + } + r = await client.post( + f"{MODLY_API}/optimize/arrange", + json={ + "path": arguments["path"], + "steps": arguments.get("steps") or [], + "params": {"normalize_scale": scale_params} if scale_params else {}, + }, + ) + r.raise_for_status() + data = r.json() + dims = " × ".join(f"{d:.4g}" for d in data.get("dimensions", [])) + text = ( + f"Applied {' → '.join(data.get('applied', []))}. " + f"New mesh path: {data['path']}. Dimensions (x × y × z, metres): {dims}." + ) + return text, {"type": "mesh_update", "url": data["url"]} + elif name == "decimate_mesh": r = await client.post( f"{MODLY_API}/optimize/mesh", @@ -411,19 +634,6 @@ class AgentChatResponse(BaseModel): thinking: str | None = None -def _extract_thinking(msg: dict) -> tuple[str, str | None]: - """Return (clean_content, thinking_text). Handles llama-server's - reasoning_content field and inline tags.""" - content = msg.get("content") or "" - thinking = msg.get("reasoning_content") or None - if not thinking: - match = re.search(r"(.*?)", content, re.DOTALL) - if match: - thinking = match.group(1).strip() - content = (content[: match.start()] + content[match.end() :]).strip() - return content, thinking - - def _auth_headers(base_url: str, api_key: Optional[str]) -> dict: headers: dict[str, str] = {} if api_key: @@ -518,8 +728,198 @@ def _drop_images(messages: list[dict]) -> bool: return dropped -@router.post("/chat", response_model=AgentChatResponse) -async def agent_chat(request: AgentChatRequest): +class _ThinkSplitter: + """Route inline … text to the reasoning, the rest to the + answer, holding back a tag a chunk boundary has cut in two.""" + + _OPEN, _CLOSE = "", "" + + def __init__(self) -> None: + self._buffer = "" + self._inside = False + + def _held(self, *tags: str) -> int: + """Length of the buffer's tail that may be the start of a tag.""" + return max( + next((n for n in range(len(tag) - 1, 0, -1) if self._buffer.endswith(tag[:n])), 0) + for tag in tags + ) + + def _cut(self, index: int, tag: str, kind: str, parts: list) -> None: + if index: + parts.append((kind, self._buffer[:index])) + self._buffer = self._buffer[index + len(tag):] + + def feed(self, text: str) -> list[tuple[str, str]]: + self._buffer += text + parts: list[tuple[str, str]] = [] + while self._buffer: + close_at = self._buffer.find(self._CLOSE) + if self._inside: + if close_at >= 0: + self._cut(close_at, self._CLOSE, "thinking", parts) + self._inside = False + continue + keep = self._held(self._CLOSE) + kind = "thinking" + else: + open_at = self._buffer.find(self._OPEN) + if close_at >= 0 and (open_at < 0 or close_at < open_at): + # A chat template that opens in the prompt leaves the + # model only the closing tag: what came before it was reasoning. + self._cut(close_at, self._CLOSE, "thinking", parts) + continue + if open_at >= 0: + self._cut(open_at, self._OPEN, "token", parts) + self._inside = True + continue + keep = self._held(self._OPEN, self._CLOSE) + kind = "token" + if len(self._buffer) > keep: + parts.append((kind, self._buffer[: len(self._buffer) - keep])) + self._buffer = self._buffer[len(self._buffer) - keep:] + break + return parts + + def flush(self) -> list[tuple[str, str]]: + rest, self._buffer = self._buffer, "" + return [("thinking" if self._inside else "token", rest)] if rest else [] + + +class _Round: + """One LLM call of the tool loop, assembled from its streamed deltas.""" + + def __init__(self, answered_before: bool, thought_before: bool) -> None: + self.raw = "" # content as the model sent it, tags included + self.answer = "" + self.thinking = "" + self.tool_calls: dict[int, dict] = {} + self._splitter = _ThinkSplitter() + self._answered_before = answered_before + self._thought_before = thought_before + + def _text(self, kind: str, text: str) -> dict | None: + # Each round's text is stripped at its start, as the non-streamed + # answer was, and set apart from what earlier rounds already showed. + if kind == "token": + if not self.answer: + text = text.lstrip() + if not text: + return None + self.answer = text + return {"type": "token", "text": ("\n\n" if self._answered_before else "") + text} + self.answer += text + return {"type": "token", "text": text} + if not self.thinking: + text = text.lstrip() + if not text: + return None + self.thinking = text + return {"type": "thinking", "text": ("\n\n---\n\n" if self._thought_before else "") + text} + self.thinking += text + return {"type": "thinking", "text": text} + + def apply(self, delta: dict) -> list[dict]: + events = [] + # llama-server sends reasoning_content; Ollama and OpenRouter, reasoning. + reasoning = delta.get("reasoning_content") or delta.get("reasoning") + if isinstance(reasoning, str) and reasoning: + events.append(self._text("thinking", reasoning)) + content = delta.get("content") + if isinstance(content, str) and content: + self.raw += content + events.extend(self._text(kind, text) for kind, text in self._splitter.feed(content)) + for position, fragment in enumerate(delta.get("tool_calls") or []): + # Streamed tool calls arrive in fragments keyed by index: the id and + # name come first, the JSON arguments in pieces after them. + call = self.tool_calls.setdefault( + fragment.get("index", position), + {"id": "", "type": "function", "function": {"name": "", "arguments": ""}}, + ) + call["id"] = fragment.get("id") or call["id"] + function = fragment.get("function") or {} + call["function"]["name"] += function.get("name") or "" + arguments = function.get("arguments") + if isinstance(arguments, dict): + arguments = json.dumps(arguments) + call["function"]["arguments"] += arguments or "" + return [e for e in events if e] + + def finish(self) -> list[dict]: + return [e for e in (self._text(k, t) for k, t in self._splitter.flush()) if e] + + def calls(self) -> list[dict]: + calls = [self.tool_calls[i] for i in sorted(self.tool_calls)] + for call in calls: + # The tool result must answer the call's id; a few servers omit it. + call["id"] = call["id"] or f"call_{uuid.uuid4().hex[:8]}" + return calls + + +async def _read_round(response: httpx.Response, round_: _Round): + """Yield the answer and reasoning of one LLM call as they stream in.""" + if "text/event-stream" not in response.headers.get("content-type", ""): + # A server that ignored "stream": true answers in one JSON body. + message = json.loads(await response.aread())["choices"][0]["message"] + for event in round_.apply(message): + yield event + else: + async for line in response.aiter_lines(): + if not line.startswith("data:"): + continue + data = line[len("data:"):].strip() + if data == "[DONE]": + break + try: + choices = json.loads(data).get("choices") or [] + except ValueError: + continue + if choices: + for event in round_.apply(choices[0].get("delta") or {}): + yield event + for event in round_.finish(): + yield event + + +def _done(message: str, actions: list[ActionDone], thinking: list[str], error: bool = False) -> dict: + return { + "type": "done", + "message": message, + "actions": [a.model_dump() for a in actions], + "thinking": "\n\n---\n\n".join(thinking) if thinking else None, + # The message reports a failure rather than answering: the chat shows + # it after whatever text already streamed. + "error": error, + } + + +def _release_claimed(claim: asyncio.Future) -> None: + if not claim.cancelled() and claim.exception() is None: + claim.result().release() + + +async def _claim_slot(model: str, spec: dict): + """Hold a local LLM slot, without leaking it if the caller is cancelled. + + A streamed chat is cancelled when its client leaves, possibly while the + model is still loading. The loading thread can't be stopped and claims the + slot anyway; nobody would release it, and a held slot is never reaped. + """ + claim = asyncio.ensure_future(asyncio.to_thread(llama_pool.ensure, model, spec, True)) + try: + return await asyncio.shield(claim) + except asyncio.CancelledError: + claim.add_done_callback(_release_claimed) + raise + + +async def _chat_events(request: AgentChatRequest): + """Run the tool loop, yielding its events as they happen. + + token / thinking — answer and reasoning text, in pieces + tool_start / tool_done — around each tool call + done — always last: the final answer, every action and the full reasoning + """ messages: list[dict] = [{"role": "system", "content": SYSTEM_PROMPT}] # Inject scene context so the LLM knows current state @@ -562,9 +962,10 @@ async def agent_chat(request: AgentChatRequest): spec = llm_server.resolve_model(request.model) # hold=True: the slot stays claimed for the whole loop, so neither the # idle reaper nor a model loading elsewhere can evict it mid-answer. - slot = await asyncio.to_thread(llama_pool.ensure, request.model, spec, True) + slot = await _claim_slot(request.model, spec) except Exception as e: - return AgentChatResponse(message=f"Could not start the local LLM: {e}") + yield _done(f"Could not start the local LLM: {e}", [], [], error=True) + return base_url = slot.base_url headers: dict = {} send_images = spec["vision"] @@ -573,7 +974,8 @@ async def agent_chat(request: AgentChatRequest): extra["chat_template_kwargs"] = {"enable_thinking": False} else: if not request.provider.base_url: - return AgentChatResponse(message="No provider URL configured. Set one in Settings → Agent.") + yield _done("No provider URL configured. Set one in Settings → Agent.", [], [], error=True) + return base_url = request.provider.base_url.rstrip("/") headers = _auth_headers(base_url, request.provider.api_key) @@ -582,52 +984,70 @@ async def agent_chat(request: AgentChatRequest): actions_done: list[ActionDone] = [] all_thinking: list[str] = [] - final: AgentChatResponse | None = None + final: str | None = None + error: str | None = None + answered = False try: async with httpx.AsyncClient(timeout=120.0) as client: - for _ in range(10): # max tool-call rounds - r = await client.post( - f"{base_url}/chat/completions", - headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **extra}, - ) - # An external model's vision support is unknown up front, and a - # text-only one rejects image parts with a 400 that failed the - # whole turn. Retry once with the images described as omitted. - if r.status_code == 400 and request.provider.type != "local" and _drop_images(messages): - r = await client.post( + + async def post() -> httpx.Response: + return await client.send( + client.build_request( + "POST", f"{base_url}/chat/completions", headers=headers, - json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": False, **extra}, - ) - - if r.status_code != 200: - return AgentChatResponse(message=f"LLM error ({r.status_code}): {_error_detail(r)}") + json={"model": request.model, "messages": messages, "tools": TOOLS, "stream": True, **extra}, + ), + stream=True, + ) - msg = r.json()["choices"][0]["message"] - tool_calls = msg.get("tool_calls") or [] - assistant: dict = {"role": "assistant", "content": msg.get("content") or ""} + for _ in range(10): # max tool-call rounds + round_ = _Round(answered, bool(all_thinking)) + r = await post() + try: + # An external model's vision support is unknown up front, and a + # text-only one rejects image parts with a 400 that failed the + # whole turn. Retry once with the images described as omitted. + if r.status_code == 400 and request.provider.type != "local" and _drop_images(messages): + await r.aclose() + r = await post() + + if r.status_code != 200: + await r.aread() + error = f"LLM error ({r.status_code}): {_error_detail(r)}" + break + + async for event in _read_round(r, round_): + yield event + finally: + await r.aclose() + + answered = answered or bool(round_.answer) + if round_.thinking: + all_thinking.append(round_.thinking) + tool_calls = round_.calls() + assistant: dict = {"role": "assistant", "content": round_.raw} if tool_calls: assistant["tool_calls"] = tool_calls messages.append(assistant) - clean_content, thinking_text = _extract_thinking(msg) - if thinking_text: - all_thinking.append(thinking_text) - if not tool_calls: - final = AgentChatResponse(message=clean_content, actions=actions_done) + final = round_.answer.strip() break for tc in tool_calls: fn = tc["function"] - result_text, payload = await execute_tool(fn["name"], _tool_arguments(fn.get("arguments")), request.context) + arguments = _tool_arguments(fn.get("arguments")) + yield {"type": "tool_start", "id": tc["id"], "tool": fn["name"], "arguments": arguments} + result_text, payload = await execute_tool(fn["name"], arguments, request.context) actions_done.append(ActionDone(tool=fn["name"], result=result_text, payload=payload)) - messages.append({"role": "tool", "tool_call_id": tc.get("id", ""), "content": result_text}) + yield {"type": "tool_done", "id": tc["id"], "tool": fn["name"], "result": result_text, "payload": payload} + messages.append({"role": "tool", "tool_call_id": tc["id"], "content": result_text}) except httpx.HTTPError as e: - return AgentChatResponse(message=f"Could not reach the LLM at {base_url}: {e}", actions=actions_done) + error = f"Could not reach the LLM at {base_url}: {e}" finally: + # Also runs when the client disconnects mid-answer. if slot is not None: slot.release() @@ -635,7 +1055,34 @@ async def agent_chat(request: AgentChatRequest): # still holding ours would keep the agent's own model on the GPU. await _unload_llm_after_workflow(request, actions_done) - if final is None: - final = AgentChatResponse(message="Reached maximum tool iterations.", actions=actions_done) - final.thinking = "\n\n---\n\n".join(all_thinking) if all_thinking else None - return final + if error is not None: + yield _done(error, actions_done, all_thinking, error=True) + elif final is None: + yield _done("Reached maximum tool iterations.", actions_done, all_thinking, error=True) + else: + yield _done(final, actions_done, all_thinking) + + +@router.post("/chat", response_model=AgentChatResponse) +async def agent_chat(request: AgentChatRequest): + """The whole turn at once, for callers that don't read a stream.""" + done: dict = {} + async for event in _chat_events(request): + if event["type"] == "done": + done = event + return AgentChatResponse(message=done["message"], actions=done["actions"], thinking=done["thinking"]) + + +@router.post("/chat/stream") +async def agent_chat_stream(request: AgentChatRequest): + """The same turn as Server-Sent Events, one JSON event per `data:` line.""" + + async def body(): + async for event in _chat_events(request): + yield f"data: {json.dumps(event)}\n\n" + + return StreamingResponse( + body(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) diff --git a/api/routers/optimize.py b/api/routers/optimize.py index c325ff206..a362b3b6d 100644 --- a/api/routers/optimize.py +++ b/api/routers/optimize.py @@ -2,6 +2,7 @@ import os import tempfile import uuid +from contextlib import contextmanager import numpy as np import trimesh @@ -23,6 +24,7 @@ MeshOpUnavailableError, mesh_ops_registry, ) +from services.mesh_ops.transforms import apply_transforms, transform_file router = APIRouter(tags=["optimize"]) @@ -47,6 +49,12 @@ class TransformRequest(BaseModel): matrix: list[list[float]] # row-major 4x4 world transform +class ArrangeRequest(BaseModel): + path: str # format: "{collection}/{filename}" + steps: list[str] # transform op ids, applied in canonical order + params: dict[str, dict[str, object]] = Field(default_factory=dict) # per step id + + def _resolve_input_path(raw_path: str) -> Path: candidate = Path(raw_path) if candidate.is_absolute(): @@ -75,23 +83,25 @@ def _operation_output_path(input_path: Path, output_name: str) -> Path: return output_dir / output_name -def _run_operation( - operation_id: str, - input_path: Path, - params: dict[str, object], +def _operation_context( output_path: Path | None = None, preserve_visuals: bool = False, -) -> MeshOpResult: - context = MeshOpContext( +) -> MeshOpContext: + return MeshOpContext( workspace_dir=registry.WORKSPACE_DIR, temp_dir=Path(tempfile.gettempdir()), output_path=output_path, preserve_visuals=preserve_visuals, ) + + +@contextmanager +def _mesh_op_http_errors(unknown_status: int = 404): + """Translate mesh-op failures into HTTP errors.""" try: - return mesh_ops_registry.run(operation_id, input_path, params, context) + yield except MeshOpNotFoundError as exc: - raise HTTPException(404, f"Unknown mesh operation: {operation_id}") from exc + raise HTTPException(unknown_status, f"Unknown mesh operation: {exc}") from exc except FileNotFoundError as exc: raise HTTPException(404, str(exc)) from exc except (MeshOpUnavailableError, MeshOpExecutionError) as exc: @@ -100,7 +110,23 @@ def _run_operation( raise HTTPException(400, str(exc)) from exc +def _run_operation( + operation_id: str, + input_path: Path, + params: dict[str, object], + output_path: Path | None = None, + preserve_visuals: bool = False, +) -> MeshOpResult: + context = _operation_context(output_path, preserve_visuals) + with _mesh_op_http_errors(): + return mesh_ops_registry.run(operation_id, input_path, params, context) + + def _operation_response(result: MeshOpResult) -> dict[str, object]: + if result.file_path is None: + # Analysis ops (inspect_mesh, suggest_fixes) return only their report. + return dict(result.details) + output_path = result.file_path.resolve() try: relative_path = output_path.relative_to(registry.WORKSPACE_DIR.resolve()).as_posix() @@ -123,8 +149,13 @@ def list_mesh_operations(): @router.post("/op/{op_name}") def run_mesh_operation(op_name: str, body: MeshOpRequest): input_path = _resolve_input_path(body.path) + # Next to the input, like the dedicated edit routes; analysis ops ignore it. + output_path = _operation_output_path( + input_path, + f"{input_path.stem}_{op_name}_{uuid.uuid4().hex[:8]}.glb", + ) return _operation_response( - _run_operation(op_name, input_path, body.params) + _run_operation(op_name, input_path, body.params, output_path) ) @@ -182,23 +213,38 @@ def transform_mesh(body: TransformRequest): if not np.all(np.isfinite(matrix)): raise HTTPException(400, "matrix contains non-finite values") - # Keep the loaded result as-is (Scene when textured/multi-geometry) so - # apply_transform preserves materials and UVs. - loaded = trimesh.load(str(input_path)) - loaded.apply_transform(matrix) - - stem = input_path.stem - output_name = f"{stem}_xf_{uuid.uuid4().hex[:8]}.glb" - workspace = registry.WORKSPACE_DIR.resolve() - output_dir = input_path.parent if registry.is_within_workspace(input_path.resolve()) else workspace / "Workflows" - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / output_name - loaded.export(str(output_path)) + output_path = transform_file(input_path, matrix, _transform_output_path(input_path)) - rel = output_path.resolve().relative_to(workspace).as_posix() + rel = output_path.resolve().relative_to(registry.WORKSPACE_DIR.resolve()).as_posix() return {"url": f"/workspace/{rel}"} +@router.post("/arrange") +def arrange_mesh(body: ArrangeRequest): + # Semantic transforms (recenter, drop_to_ground, auto_orient, + # normalize_scale) composed into one file, in canonical order. + input_path = _resolve_input_path(body.path) + with _mesh_op_http_errors(unknown_status=400): + steps = [ + (step_id, mesh_ops_registry.resolve_params(step_id, body.params.get(step_id))) + for step_id in body.steps + ] + result = apply_transforms( + input_path, + steps, + _transform_output_path(input_path), + _operation_context(), + ) + return _operation_response(result) + + +def _transform_output_path(input_path: Path) -> Path: + return _operation_output_path( + input_path, + f"{input_path.stem}_xf_{uuid.uuid4().hex[:8]}.glb", + ) + + class ImportByPathRequest(BaseModel): path: str # absolute path on disk diff --git a/api/services/mesh_ops/builtin.py b/api/services/mesh_ops/builtin.py index 900bd1933..f8450acd8 100644 --- a/api/services/mesh_ops/builtin.py +++ b/api/services/mesh_ops/builtin.py @@ -1,6 +1,9 @@ """Built-in mesh operation definitions.""" +from .inspection import inspect_mesh, suggest_fixes from .operations import decimate_mesh, repair_mesh, smooth_mesh +from .repair import auto_repair_mesh +from .transforms import auto_orient, drop_to_ground, normalize_scale, recenter from .types import MeshOp @@ -106,6 +109,73 @@ ) +INSPECT_PARAMS = ( + { + "id": "check_self_intersections", + "label": "Check Self-Intersections", + "type": "boolean", + "default": True, + "tooltip": ( + "Detect intersecting faces. The slowest check on very dense meshes." + ), + }, +) + + +NORMALIZE_SCALE_PARAMS = ( + { + "id": "size", + "label": "Size", + "type": "float", + "default": 1.0, + "min": 0.001, + "max": 100000.0, + "step": 0.1, + "tooltip": "Target size of the measured dimension, in the chosen unit.", + }, + { + "id": "unit", + "label": "Unit", + "type": "select", + "default": "m", + "options": [ + {"value": "m", "label": "Metres"}, + {"value": "cm", "label": "Centimetres"}, + {"value": "mm", "label": "Millimetres"}, + ], + "tooltip": "glTF stores metres: 20 cm is written as 0.2 units.", + }, + { + "id": "dimension", + "label": "Dimension", + "type": "select", + "default": "longest", + "options": [ + {"value": "longest", "label": "Longest side"}, + {"value": "x", "label": "Width (X)"}, + {"value": "y", "label": "Height (Y)"}, + {"value": "z", "label": "Depth (Z)"}, + ], + "tooltip": "Which bounding-box dimension is set to the target size.", + }, +) + + +REPAIR_MESH_PARAMS = ( + { + "id": "fill_holes", + "label": "Fill Holes", + "type": "boolean", + "default": True, + "tooltip": ( + "Close every hole, whatever its size. Turn off for meshes that are " + "meant to stay open, such as terrain: only flipped faces are then " + "fixed, since removing bad faces would open new holes." + ), + }, +) + + BUILTIN_MESH_OPS = ( MeshOp( id="repair", @@ -114,6 +184,13 @@ fn=repair_mesh, category="repair", ), + MeshOp( + id="repair_mesh", + label="Repair Mesh (auto)", + params_schema=REPAIR_MESH_PARAMS, + fn=auto_repair_mesh, + category="repair", + ), MeshOp( id="decimate", label="Optimize Mesh", @@ -128,4 +205,46 @@ fn=smooth_mesh, category="optimization", ), + MeshOp( + id="inspect_mesh", + label="Inspect Mesh", + params_schema=INSPECT_PARAMS, + fn=inspect_mesh, + category="analysis", + ), + MeshOp( + id="suggest_fixes", + label="Suggest Fixes", + params_schema=INSPECT_PARAMS, + fn=suggest_fixes, + category="analysis", + ), + MeshOp( + id="recenter", + label="Recenter", + params_schema=(), + fn=recenter, + category="transform", + ), + MeshOp( + id="drop_to_ground", + label="Drop to Ground", + params_schema=(), + fn=drop_to_ground, + category="transform", + ), + MeshOp( + id="auto_orient", + label="Lay Flat", + params_schema=(), + fn=auto_orient, + category="transform", + ), + MeshOp( + id="normalize_scale", + label="Normalize Scale", + params_schema=NORMALIZE_SCALE_PARAMS, + fn=normalize_scale, + category="transform", + ), ) diff --git a/api/services/mesh_ops/geometry.py b/api/services/mesh_ops/geometry.py new file mode 100644 index 000000000..3fa3c60f3 --- /dev/null +++ b/api/services/mesh_ops/geometry.py @@ -0,0 +1,44 @@ +"""World-space geometry helpers shared by the analysis and transform ops.""" + + +def scene_meshes(loaded, trimesh_module) -> list[tuple]: + """Return (geometry, world transform or None) for every non-empty triangle mesh.""" + if not isinstance(loaded, trimesh_module.Scene): + candidates = [(loaded, None)] + else: + candidates = [] + for node in loaded.graph.nodes_geometry: + transform, geometry_name = loaded.graph[node] + candidates.append((loaded.geometry.get(geometry_name), transform)) + return [ + (geometry, transform) + for geometry, transform in candidates + if isinstance(geometry, trimesh_module.Trimesh) and len(geometry.faces) + ] + + +def merged_geometry(parts, trimesh_module, process: bool = True): + """Concatenate the parts into one world-space mesh, without visuals. + + With process (the default), vertices are merged by position: UV seams split + vertices, and without the merge every textured mesh would look open along + its seams. Pure measurements (bounds, centroid) can skip it. + """ + import numpy as np + + vertices = [] + faces = [] + offset = 0 + for geometry, transform in parts: + points = np.asarray(geometry.vertices, dtype=float) + if transform is not None: + points = trimesh_module.transformations.transform_points(points, transform) + vertices.append(points) + faces.append(np.asarray(geometry.faces, dtype=np.int64) + offset) + offset += len(points) + + return trimesh_module.Trimesh( + vertices=np.vstack(vertices), + faces=np.vstack(faces), + process=process, + ) diff --git a/api/services/mesh_ops/inspection.py b/api/services/mesh_ops/inspection.py new file mode 100644 index 000000000..fd4ed6b73 --- /dev/null +++ b/api/services/mesh_ops/inspection.py @@ -0,0 +1,322 @@ +"""Read-only mesh diagnostics: inspection report and derived fix suggestions.""" + +import json +import struct +from pathlib import Path +from typing import Any, Mapping, Optional + +from .geometry import merged_geometry, scene_meshes +from .types import ( + MeshOpContext, + MeshOpExecutionError, + MeshOpResult, + MeshOpUnavailableError, +) + + +# Above this triangle count, suggest a decimation pass (slicers and the viewer +# both struggle well before the generators' multi-million-face outputs). +HIGH_FACE_COUNT = 500_000 +DECIMATE_TARGET_FACES = 200_000 + + +def _trimesh(): + try: + import trimesh + except ImportError as exc: + raise MeshOpUnavailableError( + "inspect_mesh: trimesh is not available on this system" + ) from exc + return trimesh + + +def _has_uvs(geometry, trimesh_module) -> bool: + visual = geometry.visual + if not isinstance(visual, trimesh_module.visual.TextureVisuals): + return False + return visual.uv is not None and len(visual.uv) > 0 + + +def _gltf_has_normals(document: Mapping[str, Any]) -> bool: + primitives = [ + primitive + for mesh in document.get("meshes", []) + for primitive in mesh.get("primitives", []) + # Only triangle primitives (mode 4, the default) are inspected. + if primitive.get("mode", 4) == 4 + ] + return bool(primitives) and all( + "NORMAL" in primitive.get("attributes", {}) for primitive in primitives + ) + + +def _file_has_normals(input_path: Path) -> Optional[bool]: + """Whether the file stores vertex normals; None when the format is unknown. + + trimesh computes normals on demand, so its meshes always have them; the + file itself is the only reliable source. + """ + suffix = input_path.suffix.lower() + try: + if suffix == ".glb": + with open(input_path, "rb") as handle: + header = handle.read(20) + if len(header) < 20 or header[:4] != b"glTF": + return None + chunk_length = struct.unpack_from(" dict[str, Any]: + import numpy as np + + edges, counts = np.unique(merged.edges_sorted, axis=0, return_counts=True) + boundary = edges[counts == 1] + loops = ( + trimesh_module.graph.connected_components(boundary, min_len=1) + if len(boundary) + else [] + ) + return { + "boundary_edge_count": int(len(boundary)), + "non_manifold_edge_count": int(np.count_nonzero(counts > 2)), + "hole_count": int(len(loops)), + # A simple boundary loop has as many edges as vertices. + "largest_hole_edges": int(max((len(loop) for loop in loops), default=0)), + } + + +def topology_summary(merged, trimesh_module) -> dict[str, Any]: + """Topology checks on a position-merged mesh, cheap enough to rerun per pass.""" + topology = _edge_topology(merged, trimesh_module) + is_watertight = bool(merged.is_watertight) + is_winding_consistent = bool(merged.is_winding_consistent) + return { + "is_watertight": is_watertight, + "is_winding_consistent": is_winding_consistent, + "is_manifold": topology["non_manifold_edge_count"] == 0, + # A closed, consistently wound mesh with negative volume has every + # normal pointing inwards. + "is_inside_out": bool( + is_watertight and is_winding_consistent and merged.volume < 0 + ), + **topology, + "degenerate_face_count": int( + len(merged.faces) - int(merged.nondegenerate_faces().sum()) + ), + } + + +def topology_issues(summary: Mapping[str, Any]) -> list[str]: + issues = [] + if summary["hole_count"]: + issues.append( + f"{summary['hole_count']} hole(s) " + f"({summary['boundary_edge_count']} open boundary edges)" + ) + if summary["non_manifold_edge_count"]: + issues.append( + f"{summary['non_manifold_edge_count']} non-manifold edge(s)" + ) + if not summary["is_winding_consistent"]: + issues.append("inconsistent face winding (normals flipped on some faces)") + if summary["is_inside_out"]: + issues.append("inside-out (every normal points inwards)") + if summary["degenerate_face_count"]: + issues.append(f"{summary['degenerate_face_count']} degenerate face(s)") + return issues + + +def _self_intersecting_faces(merged, context: MeshOpContext) -> Optional[int]: + try: + import pymeshlab + except ImportError: + context.log("Self-intersection check skipped: pymeshlab is unavailable") + return None + + mesh_set = pymeshlab.MeshSet() + mesh_set.add_mesh( + pymeshlab.Mesh(vertex_matrix=merged.vertices, face_matrix=merged.faces) + ) + try: + mesh_set.compute_selection_by_self_intersections_per_face() + except Exception as exc: + context.log(f"Self-intersection check skipped: {exc}") + return None + return int(mesh_set.current_mesh().selected_face_number()) + + +def _printable(report: Mapping[str, Any]) -> Optional[bool]: + """True/False when the verdict is certain; None when it hinges on an unchecked test.""" + if not ( + report["is_watertight"] + and report["is_winding_consistent"] + and report["is_manifold"] + and not report["is_inside_out"] + ): + return False + if report["self_intersecting_face_count"] is None: + return None + return report["self_intersecting_face_count"] == 0 + + +def _issues(report: Mapping[str, Any]) -> list[str]: + issues = topology_issues(report) + if report["self_intersecting_face_count"]: + issues.append( + f"{report['self_intersecting_face_count']} self-intersecting face(s)" + ) + if report["self_intersecting_face_count"] is None: + issues.append("self-intersections were not checked") + if report["component_count"] > 1: + issues.append(f"{report['component_count']} disconnected parts") + return issues + + +def build_report( + input_path: Path, + context: MeshOpContext, + check_self_intersections: bool = True, +) -> dict[str, Any]: + """Diagnose a mesh file without modifying it; every value is JSON-safe.""" + trimesh = _trimesh() + + context.progress(10, "Loading mesh…") + loaded = trimesh.load(str(input_path), process=False) + parts = scene_meshes(loaded, trimesh) + if not parts: + raise MeshOpExecutionError( + f"inspect_mesh: {input_path.name} contains no triangle mesh" + ) + + context.progress(30, "Analysing topology…") + merged = merged_geometry(parts, trimesh) + topology = topology_summary(merged, trimesh) + + self_intersections = None + if check_self_intersections: + context.progress(60, "Checking self-intersections…") + self_intersections = _self_intersecting_faces(merged, context) + + context.progress(90, "Measuring…") + bounds = merged.bounds + report: dict[str, Any] = { + "vertex_count": int(sum(len(geometry.vertices) for geometry, _ in parts)), + "face_count": int(sum(len(geometry.faces) for geometry, _ in parts)), + "component_count": int(merged.body_count), + **topology, + "self_intersecting_face_count": self_intersections, + "bounds": { + "min": [float(value) for value in bounds[0]], + "max": [float(value) for value in bounds[1]], + }, + "dimensions": [float(value) for value in merged.extents], + "surface_area": float(merged.area), + # The signed volume of an open or mis-wound mesh is meaningless. + "volume": ( + float(abs(merged.volume)) + if topology["is_watertight"] and topology["is_winding_consistent"] + else None + ), + "has_uvs": any(_has_uvs(geometry, trimesh) for geometry, _ in parts), + "has_normals": _file_has_normals(input_path), + } + report["printable"] = _printable(report) + report["issues"] = _issues(report) + context.log( + f"Inspected {report['face_count']} faces: " + + ("printable" if report["printable"] else "; ".join(report["issues"])) + ) + context.progress(100, "Done") + return report + + +def fixes_for_report(report: Mapping[str, Any]) -> tuple[list[dict], list[str]]: + """Derive registry op calls from a report, plus notes for unfixable issues.""" + fixes: list[dict] = [] + notes: list[str] = [] + + repair_reasons = [] + if report["hole_count"]: + repair_reasons.append(f"fill {report['hole_count']} hole(s)") + if report["non_manifold_edge_count"]: + repair_reasons.append("fix non-manifold edges") + if report["degenerate_face_count"]: + repair_reasons.append("remove degenerate faces") + if not report["is_winding_consistent"] or report["is_inside_out"]: + repair_reasons.append("re-orient the faces") + if repair_reasons: + fixes.append( + { + "op": "repair_mesh", + "params": {}, + "reason": "Repair to " + ", ".join(repair_reasons) + ".", + } + ) + + if report["face_count"] > HIGH_FACE_COUNT: + fixes.append( + { + "op": "decimate", + "params": {"target_faces": DECIMATE_TARGET_FACES}, + "reason": ( + f"{report['face_count']} faces is heavy; reduce to " + f"{DECIMATE_TARGET_FACES}." + ), + } + ) + + if report["self_intersecting_face_count"]: + notes.append("Self-intersections have no automatic fix yet.") + if report["component_count"] > 1: + notes.append( + f"{report['component_count']} disconnected parts; remove stray " + "fragments manually if they are not intended." + ) + return fixes, notes + + +def inspect_mesh( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + report = build_report( + input_path, + context, + check_self_intersections=bool(params.get("check_self_intersections", True)), + ) + return MeshOpResult(details=report) + + +def suggest_fixes( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + report = build_report( + input_path, + context, + check_self_intersections=bool(params.get("check_self_intersections", True)), + ) + fixes, notes = fixes_for_report(report) + return MeshOpResult(details={"report": report, "fixes": fixes, "notes": notes}) diff --git a/api/services/mesh_ops/processor.py b/api/services/mesh_ops/processor.py index 9af04353c..f31c5376d 100644 --- a/api/services/mesh_ops/processor.py +++ b/api/services/mesh_ops/processor.py @@ -56,6 +56,10 @@ def run_processor(operation_id: str, processor_id: str) -> None: data.get("params") or {}, context, ) + if result.file_path is None: + raise ValueError( + f"{processor_id}: {operation_id} produces no mesh file" + ) _emit( { "type": "done", diff --git a/api/services/mesh_ops/registry.py b/api/services/mesh_ops/registry.py index 5d7553400..92c226df3 100644 --- a/api/services/mesh_ops/registry.py +++ b/api/services/mesh_ops/registry.py @@ -95,6 +95,15 @@ def run( if not path.is_file(): raise FileNotFoundError(f"Input mesh not found: {path}") + return operation.fn(path, self.resolve_params(operation_id, params), context) + + def resolve_params( + self, + operation_id: str, + params: Optional[Mapping[str, Any]], + ) -> dict[str, Any]: + """Apply an operation's schema defaults, then validate and clamp params.""" + operation = self.get(operation_id) resolved_params = { schema["id"]: deepcopy(schema["default"]) for schema in operation.params_schema @@ -113,4 +122,4 @@ def run( schema, ) - return operation.fn(path, resolved_params, context) + return resolved_params diff --git a/api/services/mesh_ops/repair.py b/api/services/mesh_ops/repair.py new file mode 100644 index 000000000..d964b310a --- /dev/null +++ b/api/services/mesh_ops/repair.py @@ -0,0 +1,213 @@ +"""Composite, diagnostic-driven repair: loop until the mesh is closed and manifold.""" + +from pathlib import Path +from typing import Any, Mapping + +from .geometry import merged_geometry, scene_meshes +from .inspection import _has_uvs, topology_issues, topology_summary +from .operations import _mesh_libraries, _output_path +from .types import MeshOpContext, MeshOpExecutionError, MeshOpResult + + +# Filling a hole can leave a non-manifold edge behind, and fixing that edge can +# open a new hole, so a single pass is not enough; three settle every case seen +# in practice, and the loop also stops as soon as a pass makes no progress. +MAX_PASSES = 3 + + +def _is_sound(summary: Mapping[str, Any], fill_holes: bool) -> bool: + # Inside-out is left to the final flip: it needs a closed, coherent mesh. + return ( + (summary["is_watertight"] or not fill_holes) + and summary["is_manifold"] + and summary["is_winding_consistent"] + ) + + +def _progress_key(summary: Mapping[str, Any]) -> tuple: + return ( + summary["hole_count"], + summary["boundary_edge_count"], + summary["non_manifold_edge_count"], + summary["is_winding_consistent"], + summary["degenerate_face_count"], + ) + + +def _merged(vertices, faces, trimesh_module): + # Merged by position exactly as inspect_mesh does, so each pass is judged + # by the same diagnostic the user will see afterwards. + return trimesh_module.Trimesh(vertices=vertices, faces=faces, process=True) + + +def _run_pass( + mesh, + summary: Mapping[str, Any], + fill_holes: bool, + context: MeshOpContext, + pymeshlab_module, + trimesh_module, +) -> tuple[Any, list[str]]: + """Run the repair steps the diagnostic calls for; return the mesh and steps. + + Removing degenerate or non-manifold faces opens holes, so those steps only + run when the holes may be filled; otherwise the repair would leave the mesh + more open than it found it. + """ + steps = [] + if fill_holes and summary["degenerate_face_count"]: + # The holes this opens are closed by the fill below. + mesh.update_faces(mesh.nondegenerate_faces()) + mesh.remove_unreferenced_vertices() + steps.append("remove_degenerate") + + mesh_set = pymeshlab_module.MeshSet() + mesh_set.add_mesh( + pymeshlab_module.Mesh(vertex_matrix=mesh.vertices, face_matrix=mesh.faces) + ) + mesh_set.meshing_remove_duplicate_faces() + if fill_holes and not summary["is_manifold"]: + mesh_set.meshing_repair_non_manifold_edges(method=0) + mesh_set.meshing_repair_non_manifold_vertices() + mesh_set.meshing_remove_unreferenced_vertices() + steps.append("fix_non_manifold") + + if fill_holes and (summary["hole_count"] or steps): + # No size limit: a hole's boundary can't have more edges than the mesh. + max_hole_size = 3 * mesh_set.current_mesh().face_number() + 3 + try: + mesh_set.meshing_close_holes( + maxholesize=max_hole_size, + newfaceselected=False, + selfintersection=False, + ) + steps.append("fill_holes") + except Exception as exc: + context.log(f"Hole fill skipped this pass: {exc}") + + current = mesh_set.current_mesh() + mesh = _merged(current.vertex_matrix(), current.face_matrix(), trimesh_module) + if not mesh.is_winding_consistent: + # pymeshlab rather than trimesh.repair.fix_winding, which needs + # networkx: not one of Modly's dependencies. + mesh_set = pymeshlab_module.MeshSet() + mesh_set.add_mesh( + pymeshlab_module.Mesh(vertex_matrix=mesh.vertices, face_matrix=mesh.faces) + ) + try: + mesh_set.meshing_re_orient_faces_coherently() + except Exception as exc: + # Orientation needs a manifold mesh; the next pass fixes that first. + context.log(f"Face re-orientation skipped this pass: {exc}") + else: + current = mesh_set.current_mesh() + mesh = _merged(current.vertex_matrix(), current.face_matrix(), trimesh_module) + steps.append("reorient_faces") + return mesh, steps + + +def _unchanged(before: Mapping[str, Any], mesh, context: MeshOpContext) -> MeshOpResult: + """Report a mesh the repair leaves as it is, without writing a file.""" + remaining = topology_issues(before) + context.log( + "Nothing repaired" + (f": {'; '.join(remaining)}" if remaining else "") + ) + context.progress(100, "Done") + return MeshOpResult( + details={ + "repaired": False, + "passes": 0, + "steps": [], + "before": before, + "after": before, + "remaining_issues": remaining, + "face_count": int(len(mesh.faces)), + "textures_dropped": False, + } + ) + + +def auto_repair_mesh( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, +) -> MeshOpResult: + """Repair what the diagnostic finds, re-checking after every pass. + + The output is geometry only: hole filling and face removal change the + topology, so the original UVs and textures cannot be carried over. + """ + pymeshlab, trimesh = _mesh_libraries("repair_mesh") + fill_holes = bool(params.get("fill_holes", True)) + + context.progress(10, "Diagnosing…") + loaded = trimesh.load(str(input_path), process=False) + parts = scene_meshes(loaded, trimesh) + if not parts: + raise MeshOpExecutionError( + f"repair_mesh: {input_path.name} contains no triangle mesh" + ) + # Merged by position, so UV seams are not mistaken for holes. + mesh = merged_geometry(parts, trimesh) + before = topology_summary(mesh, trimesh) + has_textures = any(_has_uvs(geometry, trimesh) for geometry, _ in parts) + + if ( + _is_sound(before, fill_holes) + and not before["is_inside_out"] + and not before["degenerate_face_count"] + ): + return _unchanged(before, mesh, context) + + summary = before + steps: list[str] = [] + passes = 0 + for passes in range(1, MAX_PASSES + 1): + context.progress(20 + 20 * (passes - 1), f"Repair pass {passes}…") + previous = summary + mesh, pass_steps = _run_pass( + mesh, summary, fill_holes, context, pymeshlab, trimesh + ) + steps.extend(pass_steps) + summary = topology_summary(mesh, trimesh) + context.log( + f"Pass {passes}: " + + ("; ".join(topology_issues(summary)) or "closed and manifold") + ) + if _is_sound(summary, fill_holes) or _progress_key(summary) == _progress_key(previous): + break + + if summary["is_inside_out"]: + mesh.invert() + steps.append("flip_normals") + summary = topology_summary(mesh, trimesh) + + if not steps: + # Only reachable with fill_holes off: every remaining issue needs a fill. + return _unchanged(before, mesh, context) + if not len(mesh.faces): + raise MeshOpExecutionError("repair_mesh: the repair left no faces") + + context.progress(90, "Exporting…") + output_path = _output_path(context, "repair-mesh") + mesh.export(str(output_path)) + remaining = topology_issues(summary) + context.log( + f"Repaired in {passes} pass(es): " + + ("; ".join(remaining) if remaining else "closed and manifold") + ) + context.progress(100, "Done") + return MeshOpResult( + file_path=output_path, + details={ + "repaired": True, + "passes": passes, + # In first-applied order, without repeats across passes. + "steps": list(dict.fromkeys(steps)), + "before": before, + "after": summary, + "remaining_issues": remaining, + "face_count": int(len(mesh.faces)), + "textures_dropped": has_textures, + }, + ) diff --git a/api/services/mesh_ops/transforms.py b/api/services/mesh_ops/transforms.py new file mode 100644 index 000000000..12266f115 --- /dev/null +++ b/api/services/mesh_ops/transforms.py @@ -0,0 +1,195 @@ +"""Semantic transforms: derive a 4x4 matrix from the mesh itself, then bake it.""" + +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .geometry import merged_geometry, scene_meshes +from .operations import _output_path +from .types import ( + MeshOpContext, + MeshOpExecutionError, + MeshOpResult, + MeshOpUnavailableError, +) + + +# glTF's unit is the metre. +UNIT_SCALE = {"m": 1.0, "cm": 0.01, "mm": 0.001} +DIMENSION_AXES = {"x": 0, "y": 1, "z": 2} +# Steps always run in this order, whatever order they were requested in: +# orienting or scaling after grounding would lift the mesh off the ground again. +CANONICAL_ORDER = ("auto_orient", "normalize_scale", "recenter", "drop_to_ground") +# auto_orient only acts when the thinnest PCA extent is clearly thinner than +# the next one; a cube or a sphere has no flat side to lay down on. +FLATNESS_RATIO = 0.8 + + +def _trimesh(): + try: + import trimesh + except ImportError as exc: + raise MeshOpUnavailableError( + "mesh transforms: trimesh is not available on this system" + ) from exc + return trimesh + + +def _about(pivot, matrix, trimesh_module): + """Conjugate a matrix so it acts around pivot instead of the origin.""" + translate = trimesh_module.transformations.translation_matrix + return translate(pivot) @ matrix @ translate(-pivot) + + +def _recenter(mesh, params, trimesh_module): + return trimesh_module.transformations.translation_matrix(-mesh.centroid) + + +def _drop_to_ground(mesh, params, trimesh_module): + return trimesh_module.transformations.translation_matrix( + [0.0, -mesh.bounds[0][1], 0.0] + ) + + +def _auto_orient(mesh, params, trimesh_module): + """Smallest rotation that turns the mesh's thinnest axis (PCA) vertical.""" + import numpy as np + + weights = mesh.area_faces + if weights.sum() <= 0: + return np.eye(4) + centers = mesh.triangles_center + offsets = centers - np.average(centers, axis=0, weights=weights) + covariance = (offsets * weights[:, None]).T @ offsets / weights.sum() + # eigh returns ascending eigenvalues: column 0 is the thinnest direction. + variances, axes = np.linalg.eigh(covariance) + spreads = np.sqrt(np.clip(variances, 0.0, None)) + if spreads[1] <= 0 or spreads[0] >= FLATNESS_RATIO * spreads[1]: + return np.eye(4) + + thin_axis = axes[:, 0] + # Of the two directions along the axis, take the one already pointing up so + # the rotation stays under 90° and the mesh is not flipped over. + if thin_axis[1] < 0: + thin_axis = -thin_axis + rotation = trimesh_module.geometry.align_vectors(thin_axis, [0.0, 1.0, 0.0]) + return _about(mesh.centroid, rotation, trimesh_module) + + +def _normalize_scale(mesh, params, trimesh_module): + target = float(params["size"]) * UNIT_SCALE[params["unit"]] + extents = mesh.extents + dimension = params["dimension"] + current = ( + float(max(extents)) + if dimension == "longest" + else float(extents[DIMENSION_AXES[dimension]]) + ) + if current <= 0: + raise MeshOpExecutionError( + f"normalize_scale: the mesh has no extent along {dimension}" + ) + # Scaling around the origin keeps a recentered or grounded mesh so. + return trimesh_module.transformations.scale_matrix(target / current) + + +STEP_MATRICES = { + "recenter": _recenter, + "drop_to_ground": _drop_to_ground, + "auto_orient": _auto_orient, + "normalize_scale": _normalize_scale, +} + + +def _world_mesh(loaded, trimesh_module): + parts = scene_meshes(loaded, trimesh_module) + if not parts: + raise MeshOpExecutionError("mesh transforms: the file contains no triangle mesh") + # Measurements only: no need to pay for a vertex merge. + return merged_geometry(parts, trimesh_module, process=False) + + +def transform_file(input_path: Path, matrix, output_path: Path) -> Path: + """Bake a 4x4 world transform into a mesh file, keeping materials and UVs.""" + trimesh = _trimesh() + # Keep the loaded result as-is (Scene when textured/multi-geometry) so + # apply_transform preserves materials and UVs. + loaded = trimesh.load(str(input_path)) + loaded.apply_transform(matrix) + output_path.parent.mkdir(parents=True, exist_ok=True) + loaded.export(str(output_path)) + return output_path + + +def apply_transforms( + input_path: Path, + steps: Sequence[tuple[str, Mapping[str, Any]]], + output_path: Path, + context: MeshOpContext, +) -> MeshOpResult: + """Apply semantic steps in canonical order and write a single output file. + + Each step measures the mesh as the previous steps left it, so "lay it flat, + centre it and put it on the ground" yields one consistent result. + """ + trimesh = _trimesh() + unknown = [step_id for step_id, _ in steps if step_id not in STEP_MATRICES] + if unknown: + raise ValueError(f"Unknown transform step(s): {', '.join(unknown)}") + if not steps: + raise ValueError("At least one transform step is required") + requested = dict(steps) + ordered = [step_id for step_id in CANONICAL_ORDER if step_id in requested] + + context.progress(10, "Loading mesh…") + loaded = trimesh.load(str(input_path)) + + for index, step_id in enumerate(ordered): + context.progress(20 + 60 * index // len(ordered), f"Applying {step_id}…") + matrix = STEP_MATRICES[step_id]( + _world_mesh(loaded, trimesh), + requested[step_id], + trimesh, + ) + loaded.apply_transform(matrix) + + context.progress(85, "Exporting…") + output_path.parent.mkdir(parents=True, exist_ok=True) + loaded.export(str(output_path)) + + result = _world_mesh(loaded, trimesh) + context.log(f"Applied {', '.join(ordered)} → {output_path}") + context.progress(100, "Done") + return MeshOpResult( + file_path=output_path, + details={ + "applied": ordered, + "dimensions": [float(value) for value in result.extents], + "bounds": { + "min": [float(value) for value in result.bounds[0]], + "max": [float(value) for value in result.bounds[1]], + }, + }, + ) + + +def _single_step_op(step_id: str): + def run( + input_path: Path, + params: Mapping[str, Any], + context: MeshOpContext, + ) -> MeshOpResult: + return apply_transforms( + input_path, + [(step_id, params)], + _output_path(context, step_id.replace("_", "-")), + context, + ) + + run.__name__ = step_id + return run + + +recenter = _single_step_op("recenter") +drop_to_ground = _single_step_op("drop_to_ground") +auto_orient = _single_step_op("auto_orient") +normalize_scale = _single_step_op("normalize_scale") diff --git a/api/services/mesh_ops/types.py b/api/services/mesh_ops/types.py index 5951bfe12..1a1929db6 100644 --- a/api/services/mesh_ops/types.py +++ b/api/services/mesh_ops/types.py @@ -44,9 +44,12 @@ def log(self, message: str) -> None: @dataclass(frozen=True) class MeshOpResult: - """The file produced by an operation and optional JSON-safe measurements.""" + """The file produced by an operation and optional JSON-safe measurements. - file_path: Path + Read-only (analysis) operations produce no file and leave file_path None. + """ + + file_path: Optional[Path] = None details: Mapping[str, Any] = field(default_factory=dict) diff --git a/api/tests/test_agent_router.py b/api/tests/test_agent_router.py index ecd2ed7c1..25a1bf96f 100644 --- a/api/tests/test_agent_router.py +++ b/api/tests/test_agent_router.py @@ -1,4 +1,5 @@ import asyncio +import json import unittest from unittest import mock @@ -49,5 +50,188 @@ def handler(request: httpx.Request) -> httpx.Response: self.assertIsNone(payload) +class InspectMeshToolTests(unittest.TestCase): + REPORT = { + "printable": False, + "face_count": 11, + "vertex_count": 8, + "component_count": 1, + "is_watertight": False, + "is_manifold": True, + "is_winding_consistent": True, + "hole_count": 1, + "boundary_edge_count": 3, + "non_manifold_edge_count": 0, + "degenerate_face_count": 0, + "self_intersecting_face_count": None, + "dimensions": [1.0, 2.0, 0.5], + "surface_area": 5.5, + "volume": None, + "has_uvs": False, + "has_normals": True, + "issues": ["1 hole(s) (3 open boundary edges)"], + } + + def _run(self, name: str, response: dict) -> tuple[str, list]: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=response) + + factory = _MockClientFactory(handler) + with mock.patch.object(agent.httpx, "AsyncClient", factory): + text, payload = asyncio.run( + agent.execute_tool(name, {"path": "Default/mesh.glb"}, {}) + ) + self.assertIsNone(payload) + return text, requests + + def test_inspect_mesh_calls_the_registry_op_and_summarises(self) -> None: + text, requests = self._run("inspect_mesh", self.REPORT) + + self.assertEqual(requests[0].url.path, "/optimize/op/inspect_mesh") + self.assertIn("Printable: no", text) + self.assertIn("Watertight: no", text) + self.assertIn("Self-intersecting faces: not checked", text) + self.assertIn("volume: n/a", text) + self.assertIn("Issues: 1 hole(s)", text) + + def test_suggest_fixes_lists_ops_and_notes(self) -> None: + text, requests = self._run( + "suggest_fixes", + { + "report": self.REPORT, + "fixes": [ + {"op": "repair", "params": {"fill_holes": True}, "reason": "Repair to fill 1 hole(s)."} + ], + "notes": ["2 disconnected parts"], + }, + ) + + self.assertEqual(requests[0].url.path, "/optimize/op/suggest_fixes") + self.assertIn('- repair {"fill_holes": true}', text) + self.assertIn("Note: 2 disconnected parts", text) + + def test_suggest_fixes_on_a_clean_mesh(self) -> None: + text, _ = self._run("suggest_fixes", {"report": {}, "fixes": [], "notes": []}) + + self.assertIn("No fixes needed", text) + + +class RepairMeshToolTests(unittest.TestCase): + def _run(self, arguments: dict, response: dict) -> tuple[str, object, list]: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=response) + + factory = _MockClientFactory(handler) + with mock.patch.object(agent.httpx, "AsyncClient", factory): + text, payload = asyncio.run(agent.execute_tool("repair_mesh", arguments, {})) + return text, payload, requests + + def test_repaired_mesh_updates_the_viewer_and_warns_about_textures(self) -> None: + text, payload, requests = self._run( + {"path": "Default/mesh.glb"}, + { + "path": "Workflows/repair-mesh-1.glb", + "url": "/workspace/Workflows/repair-mesh-1.glb", + "repaired": True, + "passes": 2, + "steps": ["fix_non_manifold", "fill_holes"], + "remaining_issues": [], + "face_count": 1276, + "textures_dropped": True, + }, + ) + + self.assertEqual(requests[0].url.path, "/optimize/op/repair_mesh") + self.assertEqual(json.loads(requests[0].content), {"path": "Default/mesh.glb", "params": {}}) + self.assertIn("now closed (watertight) and manifold", text) + self.assertIn("New mesh path: Workflows/repair-mesh-1.glb", text) + self.assertIn("textures and UVs were dropped", text) + self.assertEqual(payload["type"], "mesh_update") + self.assertEqual(payload["url"], "/workspace/Workflows/repair-mesh-1.glb") + + def test_partial_repair_reports_what_remains(self) -> None: + text, _, requests = self._run( + {"path": "Default/mesh.glb", "fill_holes": False}, + { + "path": "Workflows/repair-mesh-2.glb", + "url": "/workspace/Workflows/repair-mesh-2.glb", + "repaired": True, + "passes": 3, + "steps": ["fix_non_manifold"], + "remaining_issues": ["1 hole(s) (6 open boundary edges)"], + "face_count": 100, + "textures_dropped": False, + }, + ) + + self.assertEqual(json.loads(requests[0].content)["params"], {"fill_holes": False}) + self.assertIn("Still remaining: 1 hole(s)", text) + self.assertNotIn("textures", text) + + def test_sound_mesh_changes_nothing(self) -> None: + text, payload, _ = self._run( + {"path": "Default/mesh.glb"}, + {"repaired": False, "passes": 0, "steps": [], "remaining_issues": []}, + ) + + self.assertIn("Nothing to repair", text) + self.assertIsNone(payload) + + +class ArrangeMeshToolTests(unittest.TestCase): + def _run(self, arguments: dict) -> tuple[str, object, list]: + requests = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={ + "path": "Default/mesh_xf_1234abcd.glb", + "url": "/workspace/Default/mesh_xf_1234abcd.glb", + "applied": ["auto_orient", "recenter", "drop_to_ground"], + "dimensions": [0.2, 0.05, 0.1], + }) + + factory = _MockClientFactory(handler) + with mock.patch.object(agent.httpx, "AsyncClient", factory): + text, payload = asyncio.run(agent.execute_tool("arrange_mesh", arguments, {})) + return text, payload, requests + + def test_all_steps_go_in_one_request_and_the_new_path_is_reported(self) -> None: + text, payload, requests = self._run({ + "path": "Default/mesh.glb", + "steps": ["drop_to_ground", "recenter", "auto_orient"], + }) + + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0].url.path, "/optimize/arrange") + self.assertEqual(json.loads(requests[0].content), { + "path": "Default/mesh.glb", + "steps": ["drop_to_ground", "recenter", "auto_orient"], + "params": {}, + }) + # The path lets the model chain another tool on the result. + self.assertIn("New mesh path: Default/mesh_xf_1234abcd.glb", text) + self.assertEqual(payload, {"type": "mesh_update", "url": "/workspace/Default/mesh_xf_1234abcd.glb"}) + + def test_scale_arguments_become_normalize_scale_params(self) -> None: + _, _, requests = self._run({ + "path": "Default/mesh.glb", + "steps": ["normalize_scale"], + "size": 20, + "unit": "cm", + }) + + self.assertEqual( + json.loads(requests[0].content)["params"], + {"normalize_scale": {"size": 20, "unit": "cm"}}, + ) + + if __name__ == "__main__": unittest.main() diff --git a/api/tests/test_agent_streaming.py b/api/tests/test_agent_streaming.py new file mode 100644 index 000000000..123e93c2c --- /dev/null +++ b/api/tests/test_agent_streaming.py @@ -0,0 +1,293 @@ +import asyncio +import json +import threading +import unittest +from unittest import mock + +import httpx + +import routers.agent as agent + + +def _sse(*deltas: dict) -> httpx.Response: + """A streamed /chat/completions answer, one chunk per delta.""" + lines = [f"data: {json.dumps({'choices': [{'delta': d}]})}\n\n" for d in deltas] + lines.append("data: [DONE]\n\n") + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(lines).encode(), + ) + + +def _call(index: int, **fields) -> dict: + return {"tool_calls": [{"index": index, **fields}]} + + +_WORKFLOWS = {"workflows": [{"id": "wf1", "name": "My Workflow"}]} + + +class _Scripted: + """Patches httpx.AsyncClient to serve scripted responses in order, recording + every request body.""" + + def __init__(self, *responses: httpx.Response) -> None: + self._responses = list(responses) + self.bodies: list[dict] = [] + self._real = httpx.AsyncClient + + def _handler(self, request: httpx.Request) -> httpx.Response: + self.bodies.append(json.loads(request.content)) + return self._responses.pop(0) + + def __call__(self, *args, **kwargs): + kwargs["transport"] = httpx.MockTransport(self._handler) + return self._real(*args, **kwargs) + + +def _external(context=None) -> agent.AgentChatRequest: + return agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], + model="gpt-test", + provider=agent.ProviderConfig(type="external", base_url="https://llm.test/v1", api_key="k"), + context=context or {}, + ) + + +def _events(scripted: _Scripted, request: agent.AgentChatRequest) -> list[dict]: + async def collect(): + return [event async for event in agent._chat_events(request)] + + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + return asyncio.run(collect()) + + +def _text(events: list[dict], kind: str) -> str: + return "".join(e["text"] for e in events if e["type"] == kind) + + +class ChatEventTests(unittest.TestCase): + def test_answer_streams_token_by_token_then_done(self) -> None: + scripted = _Scripted(_sse({"content": "Hel"}, {"content": "lo"}, {"content": " there"})) + + events = _events(scripted, _external()) + + self.assertEqual([e["type"] for e in events], ["token", "token", "token", "done"]) + self.assertEqual(_text(events, "token"), "Hello there") + self.assertEqual(events[-1]["message"], "Hello there") + self.assertFalse(events[-1]["error"]) + self.assertTrue(scripted.bodies[0]["stream"]) + + def test_reasoning_field_streams_as_thinking(self) -> None: + events = _events( + _Scripted(_sse({"reasoning_content": "Let me "}, {"reasoning_content": "think."}, {"content": "Hi"})), + _external(), + ) + + self.assertEqual(_text(events, "thinking"), "Let me think.") + self.assertEqual(_text(events, "token"), "Hi") + self.assertEqual(events[-1]["thinking"], "Let me think.") + + def test_inline_think_tags_cut_across_chunks_are_split_out(self) -> None: + events = _events( + _Scripted(_sse({"content": "plan\n\nHello "})), + _external(), + ) + + self.assertEqual(_text(events, "thinking"), "plan") + # A "<" that turns out not to open a tag is still part of the answer. + self.assertEqual(_text(events, "token"), "Hello ") + self.assertEqual(events[-1]["message"], "Hello ") + + def test_fragmented_tool_call_is_reassembled_and_reported_around_its_run(self) -> None: + scripted = _Scripted( + _sse( + {"content": "Starting it."}, + _call(0, id="call_1", type="function", function={"name": "run_workflow", "arguments": ""}), + _call(0, function={"arguments": '{"workflow_'}), + _call(0, function={"arguments": 'id": "wf1"}'}), + ), + _sse({"content": "Done."}), + ) + + events = _events(scripted, _external(_WORKFLOWS)) + + types = [e["type"] for e in events] + self.assertEqual(types, ["token", "tool_start", "tool_done", "token", "done"]) + start, finish = events[1], events[2] + self.assertEqual((start["id"], start["tool"], start["arguments"]), ("call_1", "run_workflow", {"workflow_id": "wf1"})) + self.assertEqual(finish["payload"]["type"], "run_workflow") + # Rounds are set apart in the streamed text; the final answer is the last one. + self.assertEqual(_text(events, "token"), "Starting it.\n\nDone.") + self.assertEqual(events[-1]["message"], "Done.") + self.assertEqual([a["tool"] for a in events[-1]["actions"]], ["run_workflow"]) + + replay = scripted.bodies[1]["messages"] + assistant = next(m for m in replay if m["role"] == "assistant") + self.assertEqual(assistant["tool_calls"][0]["function"]["arguments"], '{"workflow_id": "wf1"}') + self.assertEqual(replay[-1], {"role": "tool", "tool_call_id": "call_1", "content": finish["result"]}) + + def test_tool_call_without_an_id_still_gets_its_result_matched(self) -> None: + scripted = _Scripted( + _sse(_call(0, function={"name": "run_workflow", "arguments": '{"workflow_id": "wf1"}'})), + _sse({"content": "ok"}), + ) + + events = _events(scripted, _external(_WORKFLOWS)) + + call_id = events[0]["id"] + self.assertTrue(call_id) + tool_message = scripted.bodies[1]["messages"][-1] + self.assertEqual(tool_message["tool_call_id"], call_id) + + def test_reasoning_of_successive_rounds_is_separated(self) -> None: + scripted = _Scripted( + _sse({"reasoning_content": "first"}, _call(0, id="c", function={"name": "list_workflows", "arguments": "{}"})), + _sse({"reasoning_content": "second"}, {"content": "ok"}), + ) + + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(scripted, _external()) + + self.assertEqual(_text(events, "thinking"), "first\n\n---\n\nsecond") + self.assertEqual(events[-1]["thinking"], "first\n\n---\n\nsecond") + + def test_provider_error_ends_with_an_error_done(self) -> None: + events = _events( + _Scripted(httpx.Response(401, json={"error": {"message": "Incorrect API key provided."}})), + _external(), + ) + + self.assertEqual(len(events), 1) + self.assertTrue(events[0]["error"]) + self.assertEqual(events[0]["message"], "LLM error (401): Incorrect API key provided.") + + def test_tool_loop_limit_is_reported_as_an_error(self) -> None: + rounds = [ + _sse(_call(0, id=f"c{i}", function={"name": "list_workflows", "arguments": "{}"})) + for i in range(10) + ] + with mock.patch.object(agent, "execute_tool", mock.AsyncMock(return_value=("none", None))): + events = _events(_Scripted(*rounds), _external()) + + self.assertTrue(events[-1]["error"]) + self.assertEqual(events[-1]["message"], "Reached maximum tool iterations.") + self.assertEqual(len(events[-1]["actions"]), 10) + + def test_local_slot_is_released_when_the_client_disconnects(self) -> None: + class Slot: + base_url = "http://127.0.0.1:8791/v1" + released = 0 + + def release(self) -> None: + self.released += 1 + + slot = Slot() + request = agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], model="qwen3-4b" + ) + + async def first_event_then_disconnect(): + stream = agent._chat_events(request) + first = await stream.__anext__() + await stream.aclose() + return first + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_sse({"content": "a"}, {"content": "b"}))), \ + mock.patch.object(agent.llm_server, "resolve_model", return_value={"vision": False}), \ + mock.patch.object(agent.llama_pool, "ensure", return_value=slot), \ + mock.patch.object(agent.llama_pool, "unload_all"): + first = asyncio.run(first_event_then_disconnect()) + + self.assertEqual(first, {"type": "token", "text": "a"}) + self.assertEqual(slot.released, 1) + + def test_slot_claimed_after_the_client_left_during_loading_is_released(self) -> None: + class Slot: + base_url = "http://127.0.0.1:8791/v1" + released = 0 + + def release(self) -> None: + self.released += 1 + + slot = Slot() + loading, loaded = threading.Event(), threading.Event() + + def ensure(*_args): + loading.set() + loaded.wait(5) + return slot + + request = agent.AgentChatRequest( + messages=[agent.ChatMessage(role="user", content="hi")], model="qwen3-4b" + ) + + async def leave_while_loading(): + turn = asyncio.ensure_future(agent._chat_events(request).__anext__()) + await asyncio.to_thread(loading.wait, 5) + turn.cancel() + with self.assertRaises(asyncio.CancelledError): + await turn + loaded.set() # the model finishes loading after the client left + for _ in range(200): + if slot.released: + break + await asyncio.sleep(0.01) + + with mock.patch.object(agent.llm_server, "resolve_model", return_value={"vision": False}), \ + mock.patch.object(agent.llama_pool, "ensure", ensure): + asyncio.run(leave_while_loading()) + + self.assertEqual(slot.released, 1) + + +class StreamRouteTests(unittest.TestCase): + def test_stream_route_sends_one_json_event_per_data_line(self) -> None: + async def read_body(): + response = await agent.agent_chat_stream(_external()) + self.assertEqual(response.media_type, "text/event-stream") + return "".join([chunk async for chunk in response.body_iterator]) + + with mock.patch.object(agent.httpx, "AsyncClient", _Scripted(_sse({"content": "Hi"}))): + body = asyncio.run(read_body()) + + blocks = [b for b in body.split("\n\n") if b] + self.assertTrue(all(b.startswith("data: ") for b in blocks)) + events = [json.loads(b[len("data: "):]) for b in blocks] + self.assertEqual([e["type"] for e in events], ["token", "done"]) + + def test_json_route_returns_the_same_turn_at_once(self) -> None: + scripted = _Scripted( + _sse(_call(0, id="call_1", function={"name": "run_workflow", "arguments": '{"workflow_id": "wf1"}'})), + _sse({"reasoning_content": "hm"}, {"content": "Running."}), + ) + with mock.patch.object(agent.httpx, "AsyncClient", scripted): + response = asyncio.run(agent.agent_chat(_external(_WORKFLOWS))) + + self.assertEqual(response.message, "Running.") + self.assertEqual(response.thinking, "hm") + self.assertEqual([a.tool for a in response.actions], ["run_workflow"]) + + +class ThinkSplitterTests(unittest.TestCase): + def test_partial_tag_is_held_until_it_resolves(self) -> None: + splitter = agent._ThinkSplitter() + + self.assertEqual(splitter.feed("a<"), [("token", "a")]) + self.assertEqual(splitter.feed("think>b"), [("thinking", "b")]) + self.assertEqual(splitter.feed(" None: + # The chat template opened in the prompt, so only its end streams. + splitter = agent._ThinkSplitter() + self.assertEqual(splitter.feed("plan\n\nAnswer"), [("thinking", "plan"), ("token", "\n\nAnswer")]) + + # Reasoning that streamed before the tag was already shown; the tag is not. + splitter = agent._ThinkSplitter() + parts = splitter.feed("planAnswer") + splitter.flush() + self.assertEqual(parts, [("token", "plan"), ("token", "Answer")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_ops_dependencies.py b/api/tests/test_mesh_ops_dependencies.py new file mode 100644 index 000000000..3b0d4517d --- /dev/null +++ b/api/tests/test_mesh_ops_dependencies.py @@ -0,0 +1,75 @@ +import re +import subprocess +import sys +import unittest +from pathlib import Path + + +API_DIR = Path(__file__).resolve().parents[1] + +# Runs every mesh-op family on a holed box with networkx unimportable: it is +# not in requirements.txt, so the shipped venv never has it, while a dev venv +# often does (torch pulls it in) and would hide the gap. +_SCRIPT = r""" +import sys, tempfile +sys.modules["networkx"] = None +from pathlib import Path +import numpy as np, trimesh +from services.mesh_ops import MeshOpContext, mesh_ops_registry as registry + +directory = Path(tempfile.mkdtemp()) +context = MeshOpContext(directory, directory) +box = trimesh.creation.box() +box.update_faces(np.arange(1, len(box.faces))) +path = directory / "open.glb" +box.export(path) + +registry.run("inspect_mesh", path, {}, context) +registry.run("suggest_fixes", path, {}, context) +repaired = registry.run("repair_mesh", path, {}, context) +assert repaired.details["after"]["is_watertight"], repaired.details + +# Flipped faces take the re-orientation path. +sphere = trimesh.creation.icosphere(subdivisions=2) +faces = sphere.faces.copy() +faces[:20] = faces[:20, ::-1] +flipped = directory / "flipped.glb" +trimesh.Trimesh(sphere.vertices, faces, process=False).export(flipped) +reoriented = registry.run("repair_mesh", flipped, {}, context) +assert "reorient_faces" in reoriented.details["steps"], reoriented.details +for op in ("recenter", "drop_to_ground", "auto_orient", "normalize_scale"): + registry.run(op, path, {}, context) +print("ok") +""" + + +class MeshOpDependencyTests(unittest.TestCase): + def test_mesh_ops_run_without_undeclared_networkx(self) -> None: + result = subprocess.run( + [sys.executable, "-c", _SCRIPT], + cwd=API_DIR, + capture_output=True, + text=True, + timeout=300, + ) + + self.assertEqual(result.returncode, 0, result.stderr[-2000:]) + self.assertEqual(result.stdout.strip().splitlines()[-1], "ok") + + def test_trimesh_graph_engine_is_a_declared_dependency(self) -> None: + # inspect_mesh and repair_mesh need scipy through trimesh, which does + # not depend on it; a dev environment usually has it anyway. + requirements = (API_DIR / "requirements.txt").read_text(encoding="utf-8") + declared = { + re.split(r"[<>=\[;\s]", line.strip(), maxsplit=1)[0].lower() + for line in requirements.splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + self.assertIn("scipy", declared) + self.assertIn("trimesh", declared) + self.assertIn("pymeshlab", declared) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_ops_inspection.py b/api/tests/test_mesh_ops_inspection.py new file mode 100644 index 000000000..1b364b488 --- /dev/null +++ b/api/tests/test_mesh_ops_inspection.py @@ -0,0 +1,268 @@ +import hashlib +import importlib.util +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +from services.mesh_ops import MeshOpContext, MeshOpExecutionError, mesh_ops_registry +from services.mesh_ops.inspection import ( + DECIMATE_TARGET_FACES, + HIGH_FACE_COUNT, + fixes_for_report, +) + + +HAS_PYMESHLAB = importlib.util.find_spec("pymeshlab") is not None + + +def _box(offset=(0.0, 0.0, 0.0)) -> trimesh.Trimesh: + mesh = trimesh.creation.box() + mesh.apply_translation(offset) + return mesh + + +class InspectMeshTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext( + workspace_dir=self.workspace, + temp_dir=self.workspace, + ) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _export(self, geometry, name="mesh.glb", **kwargs) -> Path: + path = self.workspace / name + geometry.export(str(path), **kwargs) + return path + + def _inspect(self, path: Path, **params) -> dict: + result = mesh_ops_registry.run("inspect_mesh", path, params, self.context) + self.assertIsNone(result.file_path) + return dict(result.details) + + def test_closed_box_is_printable(self) -> None: + path = self._export(_box(), include_normals=True) + + report = self._inspect(path) + + self.assertTrue(report["is_watertight"]) + self.assertTrue(report["is_winding_consistent"]) + self.assertTrue(report["is_manifold"]) + self.assertEqual(report["face_count"], 12) + self.assertEqual(report["component_count"], 1) + self.assertEqual(report["hole_count"], 0) + self.assertAlmostEqual(report["volume"], 1.0, places=6) + self.assertAlmostEqual(report["surface_area"], 6.0, places=6) + np.testing.assert_allclose(report["dimensions"], [1.0, 1.0, 1.0]) + self.assertTrue(report["has_normals"]) + self.assertFalse(report["has_uvs"]) + if HAS_PYMESHLAB: + self.assertEqual(report["self_intersecting_face_count"], 0) + self.assertTrue(report["printable"]) + self.assertEqual(report["issues"], []) + + def test_open_box_reports_its_hole_and_no_volume(self) -> None: + mesh = _box() + mesh.update_faces(np.arange(1, len(mesh.faces))) + path = self._export(mesh) + + report = self._inspect(path, check_self_intersections=False) + + self.assertFalse(report["is_watertight"]) + self.assertFalse(report["printable"]) + self.assertEqual(report["hole_count"], 1) + self.assertEqual(report["boundary_edge_count"], 3) + self.assertEqual(report["largest_hole_edges"], 3) + self.assertIsNone(report["volume"]) + self.assertIsNone(report["self_intersecting_face_count"]) + self.assertTrue(any("hole" in issue for issue in report["issues"])) + self.assertFalse(report["has_normals"]) + + def test_unchecked_self_intersections_leave_printability_unknown(self) -> None: + path = self._export(_box()) + + report = self._inspect(path, check_self_intersections=False) + + self.assertTrue(report["is_watertight"]) + self.assertIsNone(report["printable"]) + self.assertIn("self-intersections were not checked", report["issues"]) + + def test_normals_are_read_from_the_file_format(self) -> None: + cases = { + "normals.obj": (dict(include_normals=True), True), + "plain.obj": (dict(include_normals=False), False), + "normals.ply": (dict(vertex_normal=True), True), + "mesh.stl": ({}, False), + } + for name, (kwargs, expected) in cases.items(): + with self.subTest(name=name): + path = self._export(_box(), name=name, **kwargs) + report = self._inspect(path, check_self_intersections=False) + self.assertIs(report["has_normals"], expected) + + def test_uv_seams_do_not_count_as_holes(self) -> None: + mesh = _box() + mesh.unmerge_vertices() + uv = np.random.default_rng(0).random((len(mesh.vertices), 2)) + mesh.visual = trimesh.visual.TextureVisuals(uv=uv) + path = self._export(mesh) + + report = self._inspect(path, check_self_intersections=False) + + self.assertTrue(report["has_uvs"]) + self.assertEqual(report["vertex_count"], 36) + self.assertTrue(report["is_watertight"]) + self.assertEqual(report["hole_count"], 0) + + def test_scene_parts_are_counted_in_world_space(self) -> None: + scene = trimesh.Scene() + scene.add_geometry(_box(), node_name="a") + scene.add_geometry( + _box(), + node_name="b", + transform=trimesh.transformations.translation_matrix([3.0, 0.0, 0.0]), + ) + path = self._export(scene) + + report = self._inspect(path, check_self_intersections=False) + + self.assertEqual(report["component_count"], 2) + self.assertEqual(report["face_count"], 24) + np.testing.assert_allclose(report["dimensions"], [4.0, 1.0, 1.0]) + self.assertAlmostEqual(report["volume"], 2.0, places=6) + + @unittest.skipUnless(HAS_PYMESHLAB, "pymeshlab is not installed") + def test_overlapping_parts_are_self_intersecting(self) -> None: + mesh = trimesh.util.concatenate([_box(), _box((0.5, 0.3, 0.2))]) + path = self._export(mesh) + + report = self._inspect(path) + + self.assertGreater(report["self_intersecting_face_count"], 0) + self.assertFalse(report["printable"]) + + def test_inspection_never_modifies_the_input(self) -> None: + mesh = _box() + mesh.update_faces(np.arange(1, len(mesh.faces))) + path = self._export(mesh) + before = hashlib.sha256(path.read_bytes()).hexdigest() + mtime = path.stat().st_mtime_ns + + self._inspect(path) + mesh_ops_registry.run("suggest_fixes", path, {}, self.context) + + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), before) + self.assertEqual(path.stat().st_mtime_ns, mtime) + self.assertEqual([p.name for p in self.workspace.iterdir()], ["mesh.glb"]) + + def test_file_without_triangles_is_rejected(self) -> None: + path = self._export(trimesh.PointCloud(np.zeros((3, 3))), name="cloud.ply") + + with self.assertRaises(MeshOpExecutionError): + self._inspect(path) + + def test_suggest_fixes_proposes_a_repair_for_holes(self) -> None: + mesh = _box() + mesh.update_faces(np.arange(1, len(mesh.faces))) + path = self._export(mesh) + + result = mesh_ops_registry.run( + "suggest_fixes", + path, + {"check_self_intersections": False}, + self.context, + ) + + self.assertIsNone(result.file_path) + self.assertEqual(result.details["report"]["hole_count"], 1) + fixes = result.details["fixes"] + self.assertEqual([fix["op"] for fix in fixes], ["repair_mesh"]) + self.assertIn("fill 1 hole(s)", fixes[0]["reason"]) + + def test_inside_out_mesh_is_reported(self) -> None: + mesh = _box() + mesh.invert() + path = self._export(mesh) + + report = self._inspect(path, check_self_intersections=False) + + self.assertTrue(report["is_watertight"]) + self.assertTrue(report["is_inside_out"]) + self.assertFalse(report["printable"]) + self.assertIn("inside-out (every normal points inwards)", report["issues"]) + + +class FixesForReportTests(unittest.TestCase): + CLEAN = { + "face_count": 12, + "component_count": 1, + "is_winding_consistent": True, + "is_inside_out": False, + "hole_count": 0, + "largest_hole_edges": 0, + "non_manifold_edge_count": 0, + "degenerate_face_count": 0, + "self_intersecting_face_count": 0, + } + + def test_clean_mesh_needs_nothing(self) -> None: + self.assertEqual(fixes_for_report(self.CLEAN), ([], [])) + + def test_suggested_params_are_valid_for_the_registry(self) -> None: + report = { + **self.CLEAN, + "face_count": HIGH_FACE_COUNT + 1, + "hole_count": 2, + "largest_hole_edges": 50_000, + "non_manifold_edge_count": 4, + "degenerate_face_count": 1, + } + + fixes, notes = fixes_for_report(report) + + self.assertEqual([fix["op"] for fix in fixes], ["repair_mesh", "decimate"]) + self.assertEqual(fixes[1]["params"], {"target_faces": DECIMATE_TARGET_FACES}) + self.assertEqual(notes, []) + for fix in fixes: + declared = { + schema["id"] for schema in mesh_ops_registry.get(fix["op"]).params_schema + } + self.assertLessEqual(set(fix["params"]), declared) + + def test_flipped_or_inside_out_faces_call_for_a_repair(self) -> None: + for flaw in ({"is_winding_consistent": False}, {"is_inside_out": True}): + with self.subTest(flaw=flaw): + fixes, _ = fixes_for_report({**self.CLEAN, **flaw}) + self.assertEqual([fix["op"] for fix in fixes], ["repair_mesh"]) + self.assertIn("re-orient", fixes[0]["reason"]) + + def test_problems_without_an_op_become_notes(self) -> None: + report = { + **self.CLEAN, + "self_intersecting_face_count": 7, + "component_count": 3, + } + + fixes, notes = fixes_for_report(report) + + self.assertEqual(fixes, []) + self.assertEqual(len(notes), 2) + + +class InspectionRegistryTests(unittest.TestCase): + def test_analysis_ops_are_registered_as_read_only(self) -> None: + described = {op["id"]: op for op in mesh_ops_registry.describe()} + + for op_id in ("inspect_mesh", "suggest_fixes"): + self.assertEqual(described[op_id]["category"], "analysis") + self.assertFalse(described[op_id]["destructive"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_ops_registry.py b/api/tests/test_mesh_ops_registry.py index 31a777a20..333c7d89a 100644 --- a/api/tests/test_mesh_ops_registry.py +++ b/api/tests/test_mesh_ops_registry.py @@ -19,10 +19,21 @@ def test_builtin_metadata_is_serializable_and_complete(self) -> None: self.assertEqual( [description["id"] for description in descriptions], - ["repair", "decimate", "smooth"], + [ + "repair", + "repair_mesh", + "decimate", + "smooth", + "inspect_mesh", + "suggest_fixes", + "recenter", + "drop_to_ground", + "auto_orient", + "normalize_scale", + ], ) for description in descriptions: - self.assertIn(description["category"], {"repair", "optimization"}) + self.assertIn(description["category"], {"repair", "optimization", "analysis", "transform"}) self.assertFalse(description["destructive"]) self.assertTrue(description["undoable"]) self.assertIsInstance(description["params_schema"], list) diff --git a/api/tests/test_mesh_ops_repair.py b/api/tests/test_mesh_ops_repair.py new file mode 100644 index 000000000..0b1f543bf --- /dev/null +++ b/api/tests/test_mesh_ops_repair.py @@ -0,0 +1,181 @@ +import hashlib +import importlib.util +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +from services.mesh_ops import MeshOpContext, MeshOpExecutionError, mesh_ops_registry + + +HAS_PYMESHLAB = importlib.util.find_spec("pymeshlab") is not None + + +def _holed_sphere_with_fin() -> trimesh.Trimesh: + """A sphere missing a patch of faces, with a fin making one edge non-manifold.""" + sphere = trimesh.creation.icosphere(subdivisions=3) + sphere.update_faces(np.arange(20, len(sphere.faces))) + edge = sphere.edges_unique[100] + return trimesh.Trimesh( + vertices=np.vstack([sphere.vertices, [[0.0, 2.0, 0.0]]]), + faces=np.vstack([sphere.faces, [[edge[0], edge[1], len(sphere.vertices)]]]), + process=False, + ) + + +def _half_open_sphere() -> trimesh.Trimesh: + sphere = trimesh.creation.icosphere(subdivisions=4) + sphere.update_faces(sphere.triangles_center[:, 1] > -0.5) + return sphere + + +@unittest.skipUnless(HAS_PYMESHLAB, "pymeshlab is not installed") +class RepairMeshTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext( + workspace_dir=self.workspace, + temp_dir=self.workspace, + ) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _export(self, geometry, name="mesh.glb") -> Path: + path = self.workspace / name + geometry.export(str(path)) + return path + + def _repair(self, geometry, **params): + return mesh_ops_registry.run( + "repair_mesh", self._export(geometry), params, self.context + ) + + def _assert_closed_and_manifold(self, path: Path) -> dict: + # Judged by the same diagnostic the user and the agent see. + report = mesh_ops_registry.run( + "inspect_mesh", path, {"check_self_intersections": False}, self.context + ).details + self.assertTrue(report["is_watertight"], report["issues"]) + self.assertTrue(report["is_manifold"], report["issues"]) + self.assertTrue(report["is_winding_consistent"], report["issues"]) + self.assertFalse(report["is_inside_out"], report["issues"]) + return report + + def test_holed_non_manifold_mesh_becomes_watertight_and_manifold(self) -> None: + result = self._repair(_holed_sphere_with_fin()) + + details = result.details + self.assertTrue(details["repaired"]) + self.assertFalse(details["before"]["is_watertight"]) + self.assertFalse(details["before"]["is_manifold"]) + self.assertIn("fix_non_manifold", details["steps"]) + self.assertIn("fill_holes", details["steps"]) + self.assertLessEqual(details["passes"], 3) + self.assertTrue(details["after"]["is_watertight"]) + self._assert_closed_and_manifold(result.file_path) + + def test_large_hole_is_closed_without_a_size_limit(self) -> None: + result = self._repair(_half_open_sphere()) + + self.assertEqual(result.details["steps"], ["fill_holes"]) + self.assertEqual(result.details["remaining_issues"], []) + self._assert_closed_and_manifold(result.file_path) + + def test_flipped_faces_are_reoriented(self) -> None: + sphere = trimesh.creation.icosphere(subdivisions=3) + faces = sphere.faces.copy() + faces[:50] = faces[:50, ::-1] + + result = self._repair(trimesh.Trimesh(sphere.vertices, faces, process=False)) + + self.assertIn("reorient_faces", result.details["steps"]) + report = self._assert_closed_and_manifold(result.file_path) + self.assertGreater(report["volume"], 0) + + def test_inside_out_mesh_is_turned_the_right_way(self) -> None: + sphere = trimesh.creation.icosphere(subdivisions=3) + sphere.invert() + + result = self._repair(sphere) + + self.assertTrue(result.details["before"]["is_inside_out"]) + self.assertEqual(result.details["steps"], ["flip_normals"]) + self._assert_closed_and_manifold(result.file_path) + + def test_sound_mesh_is_left_alone(self) -> None: + result = self._repair(trimesh.creation.box()) + + self.assertIsNone(result.file_path) + self.assertFalse(result.details["repaired"]) + self.assertEqual(result.details["remaining_issues"], []) + self.assertEqual(sorted(p.name for p in self.workspace.iterdir()), ["mesh.glb"]) + + def test_holes_are_kept_when_filling_is_off(self) -> None: + result = self._repair(_half_open_sphere(), fill_holes=False) + + self.assertFalse(result.details["repaired"]) + self.assertTrue( + any("hole" in issue for issue in result.details["remaining_issues"]) + ) + + def test_without_hole_filling_no_face_is_removed(self) -> None: + # Fixing the fin would delete faces and open a hole that may not be + # filled, so the mesh is left as it is and the issues reported. + result = self._repair(_holed_sphere_with_fin(), fill_holes=False) + + self.assertFalse(result.details["repaired"]) + self.assertIsNone(result.file_path) + remaining = result.details["remaining_issues"] + self.assertTrue(any("non-manifold" in issue for issue in remaining)) + self.assertTrue(any("hole" in issue for issue in remaining)) + + def test_without_hole_filling_flipped_faces_are_still_fixed(self) -> None: + mesh = _half_open_sphere() + faces = mesh.faces.copy() + faces[:40] = faces[:40, ::-1] + + result = self._repair( + trimesh.Trimesh(mesh.vertices, faces, process=False), fill_holes=False + ) + + self.assertEqual(result.details["steps"], ["reorient_faces"]) + self.assertTrue(result.details["after"]["is_winding_consistent"]) + self.assertEqual(result.details["face_count"], len(faces)) + self.assertEqual(result.details["after"]["hole_count"], 1) + + def test_textured_input_reports_that_textures_are_dropped(self) -> None: + mesh = _half_open_sphere() + mesh.unmerge_vertices() + uv = np.random.default_rng(0).random((len(mesh.vertices), 2)) + mesh.visual = trimesh.visual.TextureVisuals(uv=uv) + + result = self._repair(mesh) + + # UV seams were merged, so only the real hole was filled. + self.assertEqual(result.details["before"]["hole_count"], 1) + self.assertTrue(result.details["textures_dropped"]) + self._assert_closed_and_manifold(result.file_path) + + def test_input_file_is_left_untouched(self) -> None: + path = self._export(_holed_sphere_with_fin()) + before = hashlib.sha256(path.read_bytes()).hexdigest() + + result = mesh_ops_registry.run("repair_mesh", path, {}, self.context) + + self.assertNotEqual(result.file_path, path) + self.assertEqual(result.file_path.parent, self.workspace / "Workflows") + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), before) + + def test_file_without_triangles_is_rejected(self) -> None: + path = self._export(trimesh.PointCloud(np.zeros((3, 3))), name="cloud.ply") + + with self.assertRaises(MeshOpExecutionError): + mesh_ops_registry.run("repair_mesh", path, {}, self.context) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_ops_transforms.py b/api/tests/test_mesh_ops_transforms.py new file mode 100644 index 000000000..21407209b --- /dev/null +++ b/api/tests/test_mesh_ops_transforms.py @@ -0,0 +1,214 @@ +import hashlib +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import trimesh + +from services.mesh_ops import MeshOpContext, mesh_ops_registry +from services.mesh_ops.transforms import apply_transforms + + +def _tilted_plate() -> trimesh.Trimesh: + """A 2 × 0.2 × 1 plate, tilted 30° about Z and moved off the origin.""" + plate = trimesh.creation.box(extents=[2.0, 0.2, 1.0]) + plate.apply_transform( + trimesh.transformations.rotation_matrix(np.radians(30), [0, 0, 1]) + ) + plate.apply_translation([3.0, 4.0, 5.0]) + return plate + + +def _world_bounds(path: Path) -> np.ndarray: + return trimesh.load(str(path)).bounds + + +class ApplyTransformsTests(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.workspace = Path(self._directory.name) + self.context = MeshOpContext( + workspace_dir=self.workspace, + temp_dir=self.workspace, + ) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _export(self, geometry, name="mesh.glb") -> Path: + path = self.workspace / name + geometry.export(str(path)) + return path + + def _apply(self, path: Path, *step_ids: str, **params) -> tuple[Path, dict]: + steps = [ + (step_id, mesh_ops_registry.resolve_params(step_id, params.get(step_id))) + for step_id in step_ids + ] + output = self.workspace / "out" / f"{len(list(self.workspace.rglob('*')))}.glb" + result = apply_transforms(path, steps, output, self.context) + self.assertEqual(result.file_path, output) + return output, dict(result.details) + + def test_ground_center_and_lay_flat_in_one_pass_whatever_the_order(self) -> None: + path = self._export(_tilted_plate()) + + output, details = self._apply(path, "drop_to_ground", "recenter", "auto_orient") + + self.assertEqual(details["applied"], ["auto_orient", "recenter", "drop_to_ground"]) + bounds = _world_bounds(output) + # Laid flat: the 0.2 thickness is now the height, and the long side + # keeps its heading (no spin about the vertical axis). + np.testing.assert_allclose(bounds[1] - bounds[0], [2.0, 0.2, 1.0], atol=1e-5) + self.assertAlmostEqual(bounds[0][1], 0.0, places=5) + np.testing.assert_allclose((bounds[0] + bounds[1])[[0, 2]] / 2, [0, 0], atol=1e-5) + np.testing.assert_allclose(details["dimensions"], [2.0, 0.2, 1.0], atol=1e-5) + + def test_recenter_moves_the_centroid_to_the_origin(self) -> None: + path = self._export(_tilted_plate()) + + output, _ = self._apply(path, "recenter") + + centroid = trimesh.load(str(output), force="mesh").centroid + np.testing.assert_allclose(centroid, [0, 0, 0], atol=1e-5) + + def test_drop_to_ground_only_moves_vertically(self) -> None: + path = self._export(_tilted_plate()) + before = _world_bounds(path) + + output, _ = self._apply(path, "drop_to_ground") + + after = _world_bounds(output) + self.assertAlmostEqual(after[0][1], 0.0, places=5) + np.testing.assert_allclose(after[:, [0, 2]], before[:, [0, 2]], atol=1e-5) + + def test_auto_orient_leaves_a_mesh_without_a_flat_side_alone(self) -> None: + cube = trimesh.creation.box() + cube.apply_transform( + trimesh.transformations.rotation_matrix(np.radians(20), [1, 1, 0]) + ) + path = self._export(cube) + + output, _ = self._apply(path, "auto_orient") + + np.testing.assert_allclose(_world_bounds(output), _world_bounds(path), atol=1e-5) + + def test_auto_orient_does_not_flip_a_mesh_over(self) -> None: + # A shallow bowl (open hemisphere-like dish) opening upwards. + bowl = trimesh.creation.icosphere(subdivisions=3) + bowl.update_faces(bowl.triangles_center[:, 1] < -0.3) + bowl.apply_scale([1.0, 0.3, 1.0]) + bowl.apply_transform( + trimesh.transformations.rotation_matrix(np.radians(10), [1, 0, 0]) + ) + path = self._export(bowl) + + output, _ = self._apply(path, "auto_orient") + + mesh = trimesh.load(str(output), force="mesh") + # Still opening upwards: the rim is above the bottom of the dish. + rim_height = mesh.vertices[:, 1].max() + lowest = mesh.vertices[:, 1].min() + center_height = mesh.vertices[np.argmin(np.linalg.norm(mesh.vertices[:, [0, 2]], axis=1)), 1] + self.assertLess(center_height - lowest, rim_height - center_height) + + def test_normalize_scale_to_real_units(self) -> None: + path = self._export(_tilted_plate()) + + _, longest = self._apply(path, "normalize_scale", normalize_scale={"size": 20, "unit": "cm"}) + _, height = self._apply( + path, + "normalize_scale", + normalize_scale={"size": 5, "unit": "mm", "dimension": "y"}, + ) + + self.assertAlmostEqual(max(longest["dimensions"]), 0.2, places=5) + self.assertAlmostEqual(height["dimensions"][1], 0.005, places=6) + + def test_scaling_keeps_a_grounded_mesh_on_the_ground(self) -> None: + path = self._export(_tilted_plate()) + + output, details = self._apply( + path, + "drop_to_ground", + "normalize_scale", + normalize_scale={"size": 3}, + ) + + self.assertEqual(details["applied"], ["normalize_scale", "drop_to_ground"]) + self.assertAlmostEqual(_world_bounds(output)[0][1], 0.0, places=5) + self.assertAlmostEqual(max(details["dimensions"]), 3.0, places=4) + + def test_scene_node_transforms_are_measured_in_world_space(self) -> None: + scene = trimesh.Scene() + scene.add_geometry( + trimesh.creation.box(), + node_name="raised", + transform=trimesh.transformations.translation_matrix([0.0, 7.0, 0.0]), + ) + path = self._export(scene) + + output, _ = self._apply(path, "drop_to_ground") + + self.assertAlmostEqual(_world_bounds(output)[0][1], 0.0, places=5) + + def test_textures_survive_the_transform(self) -> None: + mesh = trimesh.creation.box() + mesh.unmerge_vertices() + uv = np.random.default_rng(0).random((len(mesh.vertices), 2)) + mesh.visual = trimesh.visual.TextureVisuals(uv=uv) + path = self._export(mesh) + + output, _ = self._apply(path, "drop_to_ground") + + geometry = next(iter(trimesh.load(str(output)).geometry.values())) + self.assertIsInstance(geometry.visual, trimesh.visual.TextureVisuals) + self.assertEqual(len(geometry.visual.uv), len(uv)) + + def test_input_file_is_left_untouched(self) -> None: + path = self._export(_tilted_plate()) + before = hashlib.sha256(path.read_bytes()).hexdigest() + + output, _ = self._apply(path, "auto_orient", "recenter", "drop_to_ground") + + self.assertNotEqual(output, path) + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), before) + + def test_invalid_steps_are_rejected(self) -> None: + path = self._export(_tilted_plate()) + output = self.workspace / "never.glb" + + with self.assertRaises(ValueError): + apply_transforms(path, [], output, self.context) + with self.assertRaises(ValueError): + apply_transforms(path, [("repair", {})], output, self.context) + with self.assertRaises(ValueError): + mesh_ops_registry.resolve_params("normalize_scale", {"unit": "inch"}) + self.assertFalse(output.exists()) + + def test_registry_ops_write_a_new_file(self) -> None: + path = self._export(_tilted_plate()) + + result = mesh_ops_registry.run("drop_to_ground", path, {}, self.context) + + self.assertTrue(result.file_path.is_file()) + self.assertEqual(result.file_path.parent, self.workspace / "Workflows") + self.assertAlmostEqual(_world_bounds(result.file_path)[0][1], 0.0, places=5) + + +class TransformRegistryTests(unittest.TestCase): + def test_transform_ops_are_registered(self) -> None: + described = {op["id"]: op for op in mesh_ops_registry.describe()} + + for op_id in ("recenter", "drop_to_ground", "auto_orient", "normalize_scale"): + self.assertEqual(described[op_id]["category"], "transform") + self.assertFalse(described[op_id]["destructive"]) + self.assertEqual( + [schema["id"] for schema in described["normalize_scale"]["params_schema"]], + ["size", "unit", "dimension"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/api/tests/test_mesh_routers_workspace.py b/api/tests/test_mesh_routers_workspace.py index f03e86dff..8e57da721 100644 --- a/api/tests/test_mesh_routers_workspace.py +++ b/api/tests/test_mesh_routers_workspace.py @@ -69,6 +69,40 @@ def test_transform_writes_its_result_into_the_moved_workspace(self) -> None: written = registry.WORKSPACE_DIR / result["url"].removeprefix("/workspace/") self.assertTrue(written.is_file()) + def test_arrange_composes_steps_into_one_file_in_the_moved_workspace(self) -> None: + result = optimize_router.arrange_mesh( + optimize_router.ArrangeRequest( + path="MyColl/mesh.glb", + steps=["drop_to_ground", "normalize_scale"], + params={"normalize_scale": {"size": 50, "unit": "cm"}}, + ) + ) + self.assertEqual(result["applied"], ["normalize_scale", "drop_to_ground"]) + self.assertTrue(result["path"].startswith("MyColl/mesh_xf_")) + self.assertEqual(result["url"], f"/workspace/{result['path']}") + written = trimesh.load(str(registry.WORKSPACE_DIR / result["path"])) + self.assertAlmostEqual(written.bounds[0][1], 0.0, places=5) + self.assertAlmostEqual(max(written.extents), 0.5, places=5) + + def test_arrange_rejects_unknown_steps_and_bad_params(self) -> None: + for steps, params in ( + (["levitate"], {}), + (["repair"], {}), + ([], {}), + (["normalize_scale"], {"normalize_scale": {"unit": "inch"}}), + ): + with self.subTest(steps=steps), self.assertRaises(HTTPException) as raised: + optimize_router.arrange_mesh( + optimize_router.ArrangeRequest( + path="MyColl/mesh.glb", steps=steps, params=params, + ) + ) + self.assertEqual(raised.exception.status_code, 400) + self.assertEqual( + sorted(p.name for p in (registry.WORKSPACE_DIR / "MyColl").iterdir()), + ["mesh.glb"], + ) + def test_decimate_and_smooth_read_their_input_from_the_moved_workspace(self) -> None: # /optimize/mesh and /optimize/smooth resolve their input through this helper. resolved = optimize_router._resolve_input_path("MyColl/mesh.glb") diff --git a/api/tests/test_optimize_mesh_ops.py b/api/tests/test_optimize_mesh_ops.py index 6c702ddb6..5436da007 100644 --- a/api/tests/test_optimize_mesh_ops.py +++ b/api/tests/test_optimize_mesh_ops.py @@ -52,14 +52,11 @@ def test_generic_list_and_run_routes_use_the_shared_registry(self) -> None: ) self.assertEqual(descriptions[0]["id"], "repair") - self.assertEqual( - response, - { - "path": "Workflows/output.glb", - "url": "/workspace/Workflows/output.glb", - "face_count": 42, - }, - ) + # Written next to the input under a unique name, never over it. + self.assertRegex(response["path"], r"^input_repair_[0-9a-f]{8}\.glb$") + self.assertEqual(response["url"], f"/workspace/{response['path']}") + self.assertEqual(response["face_count"], 42) + self.assertEqual(registry.calls[0][3].output_path.parent, workspace) self.assertEqual(registry.calls[0][0:3], ( "repair", input_path, @@ -105,6 +102,25 @@ def test_legacy_routes_delegate_with_their_existing_clamps_and_names(self) -> No self.assertEqual(optimize_response["face_count"], 42) self.assertEqual(smooth_response["url"], "/workspace/model_smooth20.glb") + def test_analysis_operation_returns_only_its_report(self) -> None: + class AnalysisRegistry: + def run(self, operation_id, input_path, params, context): + return MeshOpResult(details={"printable": False, "hole_count": 1}) + + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + (workspace / "input.glb").touch() + with ( + patch.object(optimize.registry, "WORKSPACE_DIR", workspace), + patch.object(optimize, "mesh_ops_registry", AnalysisRegistry()), + ): + response = optimize.run_mesh_operation( + "inspect_mesh", + optimize.MeshOpRequest(path="input.glb"), + ) + + self.assertEqual(response, {"printable": False, "hole_count": 1}) + def test_unknown_generic_operation_is_a_404(self) -> None: class MissingRegistry: def run(self, operation_id, input_path, params, context): diff --git a/package.json b/package.json index a63727a24..e8da0f9d5 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "prepare-resources": "node scripts/download-python-embed.js", "test": "npm run test:py && npm run test:node", "test:py": "node scripts/run-pytests.mjs", - "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", + "test:node": "node --test --experimental-strip-types --experimental-loader ./scripts/node-ts-extensionless-loader.mjs src/shared/types/assetLibrary.test.ts src/areas/generate/assetLibraryProjection.test.ts src/areas/generate/assetLibraryService.test.ts src/areas/generate/assetLibraryUi.test.ts src/areas/generate/orcaSlicerLink.test.ts src/shared/utils/agentStream.test.ts electron/main/artifact-registry-service.test.ts electron/main/extension-path-guard.test.ts electron/preload/artifact-registry-preload.test.ts && node --test electron/main/*.test.mjs src/**/*.test.mjs", "package": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder", "package:mac": "cross-env CSC_IDENTITY_AUTO_DISCOVERY=false npm run build && npm run prepare-resources && electron-builder --mac --arm64", "lint": "eslint ." diff --git a/src/areas/generate/components/ChatPanel.tsx b/src/areas/generate/components/ChatPanel.tsx index 053e780a9..d6db103e2 100644 --- a/src/areas/generate/components/ChatPanel.tsx +++ b/src/areas/generate/components/ChatPanel.tsx @@ -6,11 +6,13 @@ import { useWorkflowsStore } from '@shared/stores/workflowsStore' import { useExtensionsStore } from '@shared/stores/extensionsStore' import { useWorkflowRunStore } from '@areas/workflows/workflowRunStore' import { buildAllWorkflowExtensions } from '@areas/workflows/mockExtensions' +import { readAgentEvents } from '@shared/utils/agentStream' // ─── Types ──────────────────────────────────────────────────────────────────── import type { ThinkingMode } from '@shared/stores/agentStore' import type { Workflow } from '@shared/types/electron.d' +import type { AgentAction, AgentActionPayload } from '@shared/utils/agentStream' interface Message { id: string @@ -19,19 +21,12 @@ interface Message { thinking?: string imageDataUrls?: string[] actions?: ActionDone[] + streaming?: boolean } -interface ActionDone { - tool: string - result: string - payload?: { - type: string - url?: string - face_count?: number - workflow_id?: string - workflow_name?: string - workflow?: Omit - } | null +interface ActionDone extends AgentAction { + id?: string + running?: boolean } // ─── Constants ──────────────────────────────────────────────────────────────── @@ -86,15 +81,23 @@ const TOOL_LABELS: Record = { function ActionsCard({ actions, onUndo }: { actions: ActionDone[]; onUndo?: () => void }): JSX.Element { const [expanded, setExpanded] = useState(false) const meshActions = actions.filter((a) => a.payload?.type === 'mesh_update') - const canUndo = meshActions.length > 0 && !!onUndo + const running = actions.find((a) => a.running) + const canUndo = meshActions.length > 0 && !!onUndo && !running return (
{/* Header */}
- - {actions.length} action{actions.length > 1 ? 's' : ''} performed - + {running ? ( + + + Running {running.tool.replace(/_/g, ' ')}… + + ) : ( + + {actions.length} action{actions.length > 1 ? 's' : ''} performed + + )}
{canUndo && (