diff --git a/docs/user-guide/invokeai-integration.md b/docs/user-guide/invokeai-integration.md index bd93688a..2c50d65e 100644 --- a/docs/user-guide/invokeai-integration.md +++ b/docs/user-guide/invokeai-integration.md @@ -267,6 +267,33 @@ galleries, then the image will be reused. or later; **Append Image** requires InvokeAI 6.13.5 or later. If a button you expect is missing, upgrade InvokeAI. +#### Image generation or video generation + +When PhotoMapAI is connected to an InvokeAI release with a video +generation tab, an **as a reference for** pulldown appears next to the +two buttons. It chooses where **Send Image** and **Append Image** put +the image: **image generation** (the default, described above) or +**video generation**. PhotoMapAI remembers the choice on each device. + +With **video generation** chosen, where the image lands depends on the +video model selected in InvokeAI's Video panel: + +- **Models that take reference images**, such as MiniMax H3 Ref2VA: + **Send Image** replaces the reference images (any reference videos + stay), and **Append Image** adds the image to the end of the + references. +- **Models that take a first and last frame**, such as Wan 2.2 I2V and + LTX-2: **Send Image** makes the image the first frame and clears the + last frame. **Append Image** fills the first frame if it is empty, + otherwise the last frame; once both are set, further appends are + ignored. An initial video counts as the first frame. +- **Models that take no images**, such as Wan 2.2 T2V: the image is not + placed. + +InvokeAI never switches models to make an image fit. It reports what +happened, including an image it could not place, in a notice in its +own window. + By default, the selected image is used as a reference image for an image edit model or as an IP Adapter reference image. If you wish to use it in the InvokeAI Canvas as a raster image or controlnet, go into @@ -311,7 +338,8 @@ A few situations are not surfaced inline and are worth knowing about: - **Boards dropdown is empty or stuck on *Uncategorized*** — usually means authentication failed silently. If your InvokeAI is in multi-user mode, fill in the username and password and re-enter the URL to retrigger the probe. - **Recall / Remix succeeds but no image queues in InvokeAI** — switch to InvokeAI's browser tab. The recalled parameters land in the canvas/generation panel; you still have to press *Invoke* to start the queue. - **Remix / Recall buttons missing for an InvokeAI-generated image** — the image was probably produced by an InvokeAI version PhotoMapAI does not yet recognise (only **Send Image** and **Append Image** are shown in that case). Check the *View Metadata (JSON)* link at the bottom of the drawer to confirm the metadata is present, and please [open an issue](https://github.com/lstein/PhotoMapAI/issues) so we can add support for the new schema. -- **Append Image button missing** — the configured InvokeAI backend predates the append option (it shipped in InvokeAI 6.13.5). Upgrade InvokeAI to enable it. +- **Append Image button missing** — the configured InvokeAI backend predates the append option (it shipped in InvokeAI 6.13.5). Upgrade InvokeAI to enable it. Appending to video generation does not depend on this, so the button reappears when **video generation** is chosen. +- **No "as a reference for" pulldown** — the configured InvokeAI backend cannot place images in a video generation tab, so Send / Append Image always go to image generation. ### PhotoMapAI and InvokeAI on different machines diff --git a/photomap/backend/metadata_modules/invoke_formatter.py b/photomap/backend/metadata_modules/invoke_formatter.py index b7c6ceb6..e4f4e8e1 100644 --- a/photomap/backend/metadata_modules/invoke_formatter.py +++ b/photomap/backend/metadata_modules/invoke_formatter.py @@ -105,7 +105,8 @@ def _esc(value: object) -> str: _USE_REF_BUTTON_HTML = ( '" @@ -116,16 +117,35 @@ def _esc(value: object) -> str: # InvokeAI backend whose recall endpoint understands ``?append=true``. _APPEND_REF_BUTTON_HTML = ( '" ) +# Which InvokeAI tab Send / Append Image feed. Hidden unless the backend has +# InvokeAI 7's image placement route (``invoke-video-image-supported``); the +# choice is remembered per browser by invoke-recall.js. +_REF_TARGET_HTML = ( + 'as a reference for ' + '" +) + +_REF_ROW_HTML = ( + '
' + f"{_USE_REF_BUTTON_HTML}{_APPEND_REF_BUTTON_HTML}{_REF_TARGET_HTML}" + "
" +) + -def _recall_controls_table(buttons_html: str) -> str: - """Wrap a set of recall buttons in a two-row table styled like the Image +def _recall_controls_table(rows_html: str) -> str: + """Wrap rows of recall buttons in a two-row table styled like the Image Details table: an "InvokeAI" header row above a row holding the buttons. + ``rows_html`` is one or more ``invoke-recall-buttons`` rows. The whole table carries the ``invoke-recall-controls`` class so the capability gating in ``invoke-capabilities.js`` can hide it wholesale when @@ -136,7 +156,7 @@ def _recall_controls_table(buttons_html: str) -> str: return ( '' '' - f'' + f"" "
InvokeAI
{buttons_html}
{rows_html}
" ) @@ -144,8 +164,8 @@ def _recall_controls_table(buttons_html: str) -> str: def _recall_buttons_html() -> str: """Render the recall / remix / use-ref button group shown at the bottom of the drawer.""" return _recall_controls_table( - f"{_USE_REF_BUTTON_HTML}" - f"{_APPEND_REF_BUTTON_HTML}" + f"{_REF_ROW_HTML}" + '
' '" + "
" ) @@ -201,7 +222,9 @@ def video_buttons_html(show_recall: bool) -> str: "Recall", "Recall video parameters (including seed) to InvokeAI", ) - return _recall_controls_table(buttons).replace( + return _recall_controls_table( + f'
{buttons}
' + ).replace( 'class="invoke-recall-controls"', 'class="invoke-recall-controls invoke-video-controls"', 1, @@ -216,9 +239,7 @@ def use_ref_button_html() -> str: so it is appended to non-Invoke metadata views as well, whenever an InvokeAI backend is configured. """ - return _recall_controls_table( - f"{_USE_REF_BUTTON_HTML}{_APPEND_REF_BUTTON_HTML}" - ) + return _recall_controls_table(_REF_ROW_HTML) def format_invoke_metadata( diff --git a/photomap/backend/preferences.py b/photomap/backend/preferences.py index 58133cbc..f4f14479 100644 --- a/photomap/backend/preferences.py +++ b/photomap/backend/preferences.py @@ -82,6 +82,8 @@ class UserPreferences(_CamelModel): # Metadata drawer / cluster labels show_metadata_fields: bool = True + # Which InvokeAI tab the drawer's Send / Append Image buttons feed. + invoke_ref_target: Literal["image", "video"] = "image" autotagging_enabled: bool = False # Dataset Curator panel. Ranges mirror the HTML input min/max attrs in diff --git a/photomap/backend/routers/invoke.py b/photomap/backend/routers/invoke.py index cb58333c..97a70503 100644 --- a/photomap/backend/routers/invoke.py +++ b/photomap/backend/routers/invoke.py @@ -11,7 +11,8 @@ * ``POST /invokeai/use_ref_image`` — upload the selected image to InvokeAI and then call the same recall endpoint with the uploaded image as a reference image parameter, so the next generation uses it for visual - guidance. + guidance. With ``target="video"`` the image goes to InvokeAI 7's Video + panel instead (``/api/v1/recall/video/{queue_id}/image``). * ``POST /invokeai/video/recall`` — the video counterpart of ``/recall``: send a video generation record to InvokeAI 7's ``/api/v1/recall/video/{queue_id}``. @@ -204,6 +205,15 @@ class UseRefImageRequest(BaseModel): "reference-image list instead of replacing it" ), ) + target: Literal["image", "video"] = Field( + "image", + description=( + "``image`` sends the image to InvokeAI's image generation tab; " + "``video`` sends it to InvokeAI 7's Video panel, where a model that " + "takes reference images gets it as one and a model that takes " + "frames gets it as a frame" + ), + ) queue_id: str = Field( "default", description="InvokeAI queue id to target", @@ -292,10 +302,12 @@ async def _probe_capabilities( ``/api/v1/app/version``: recall shipped in 6.13.0, append in 6.13.5. The video recall router (``/api/v1/recall/video/{queue_id}``, InvokeAI 7) - is detected from the schema only. + and its image placement route (``…/image``) are detected from the schema + only. Returns ``{"reachable": bool, "recall": bool, "append": bool, - "video_recall": bool, "source": "openapi" | "version" | "unreachable"}``. + "video_recall": bool, "video_image": bool, + "source": "openapi" | "version" | "unreachable"}``. """ openapi_url = f"{base_url.rstrip('/')}/openapi.json" @@ -306,13 +318,13 @@ async def _do(headers: dict[str, str]) -> httpx.Response: resp = await _request_with_auth_fallback(base_url, username, password, _do) if resp.status_code == 200: spec = resp.json() - recall_post = ( - spec.get("paths", {}).get("/api/v1/recall/{queue_id}", {}).get("post") - ) + paths = spec.get("paths", {}) + recall_post = paths.get("/api/v1/recall/{queue_id}", {}).get("post") video_recall = ( - spec.get("paths", {}) - .get("/api/v1/recall/video/{queue_id}", {}) - .get("post") + paths.get("/api/v1/recall/video/{queue_id}", {}).get("post") is not None + ) + video_image = ( + paths.get("/api/v1/recall/video/{queue_id}/image", {}).get("post") is not None ) if recall_post is None: @@ -321,6 +333,7 @@ async def _do(headers: dict[str, str]) -> httpx.Response: "recall": False, "append": False, "video_recall": video_recall, + "video_image": video_image, "source": "openapi", } params = { @@ -331,6 +344,7 @@ async def _do(headers: dict[str, str]) -> httpx.Response: "recall": True, "append": "append" in params, "video_recall": video_recall, + "video_image": video_image, "source": "openapi", } except (httpx.RequestError, HTTPException, ValueError) as exc: @@ -355,6 +369,7 @@ async def _do(headers: dict[str, str]) -> httpx.Response: # No release ships the video recall router yet, so there is no # version to infer it from: only the schema can advertise it. "video_recall": False, + "video_image": False, "source": "version", "version": version, } @@ -363,6 +378,7 @@ async def _do(headers: dict[str, str]) -> httpx.Response: "recall": False, "append": False, "video_recall": False, + "video_image": False, "source": "unreachable", } @@ -379,6 +395,9 @@ async def invokeai_capabilities(refresh: bool = False) -> dict: Append (an old backend would silently treat append as replace); * recall with append — all buttons. + ``video_image`` additionally reveals the drawer's choice between image and + video generation as the destination of Send / Append. + Results are cached per configured URL; pass ``?refresh=true`` to force a re-probe (used right after the settings are saved). """ @@ -393,6 +412,7 @@ async def invokeai_capabilities(refresh: bool = False) -> dict: "recall": False, "append": False, "video_recall": False, + "video_image": False, } now = time.monotonic() @@ -738,6 +758,13 @@ async def use_ref_image(request: UseRefImageRequest) -> dict: With ``append=True`` the recall is sent with ``?append=true``, which asks InvokeAI to add the image to its existing reference-image list instead of replacing it (the drawer's "Append to InvokeAI" button). + + With ``target="video"`` the uploaded image is placed in InvokeAI 7's Video + panel through ``/api/v1/recall/video/{queue_id}/image`` instead. Where it + lands depends on the panel's model, which InvokeAI reports to the user + there: a model that takes reference images gets it as one (replacing the + reference images, or joining them with ``append``); a model that takes + frames gets it as the first frame, or with ``append`` the free frame slot. """ settings = config_manager.get_invokeai_settings() base_url = settings["url"] @@ -774,7 +801,12 @@ async def use_ref_image(request: UseRefImageRequest) -> dict: password = settings["password"] board_id = settings["board_id"] - recall_url = f"{base_url.rstrip('/')}/api/v1/recall/{request.queue_id}" + if request.target == "video": + recall_url = ( + f"{base_url.rstrip('/')}/api/v1/recall/video/{request.queue_id}/image" + ) + else: + recall_url = f"{base_url.rstrip('/')}/api/v1/recall/{request.queue_id}" try: async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT) as client: @@ -795,12 +827,19 @@ async def use_ref_image(request: UseRefImageRequest) -> dict: client, base_url, image_path, username, password, board_id=board_id ) - # Deliberately omit ``strict=true`` so that the recall only - # *adds* the reference image to whatever the user already has - # set up in InvokeAI rather than resetting every other - # parameter back to defaults. - payload = {"reference_images": [{"image_name": image_name}]} - recall_params = {"append": "true"} if request.append else None + if request.target == "video": + payload = None + recall_params = { + "image_name": image_name, + "append": "true" if request.append else "false", + } + else: + # Deliberately omit ``strict=true`` so that the recall only + # *adds* the reference image to whatever the user already has + # set up in InvokeAI rather than resetting every other + # parameter back to defaults. + payload = {"reference_images": [{"image_name": image_name}]} + recall_params = {"append": "true"} if request.append else None async def _do_recall(headers: dict[str, str]) -> httpx.Response: return await client.post( @@ -838,7 +877,8 @@ async def _do_recall(headers: dict[str, str]) -> httpx.Response: result = { "success": True, - "sent": payload, + "target": request.target, + "sent": payload if payload is not None else recall_params, "uploaded_image_name": image_name, "reused_existing": reused_existing, "response": remote, diff --git a/photomap/frontend/static/css/metadata-drawer.css b/photomap/frontend/static/css/metadata-drawer.css index f9d84fdc..a6e6f2e3 100644 --- a/photomap/frontend/static/css/metadata-drawer.css +++ b/photomap/frontend/static/css/metadata-drawer.css @@ -238,7 +238,8 @@ text-align: left; } -/* The button row itself: lay the buttons out horizontally inside the cell. */ +/* A button row: lay the buttons out horizontally inside the cell. Image + tables have two rows (Send / Append Image, then Remix / Recall). */ .invoke-recall-buttons { display: flex; gap: 12px; @@ -246,6 +247,42 @@ flex-wrap: wrap; } +.invoke-recall-buttons + .invoke-recall-buttons { + margin-top: 8px; +} + +/* "as a reference for ": which InvokeAI + tab Send / Append Image feed. Only InvokeAI 7 can place an image in its + Video panel, so the choice stays hidden until the backend confirms it. */ +.invoke-ref-target { + display: none; + flex-wrap: wrap; + align-items: center; + gap: 6px; + color: #ccc; + font-size: 14px; +} + +body.invoke-video-image-supported .invoke-ref-target { + display: inline-flex; +} + +.invoke-ref-target-select { + background: #2a2a2a; + color: #faea0e; + border: 1px solid #555; + border-radius: 4px; + height: 32px; + padding: 0 6px; + font-size: 14px; + cursor: pointer; +} + +.invoke-ref-target-select:hover, +.invoke-ref-target-select:focus-visible { + border-color: #faea0e; +} + /* Capability gating (see invoke-capabilities.js): every recall button stays hidden until the backend confirms the configured InvokeAI instance supports the recall API; the Append button additionally requires the @@ -264,6 +301,14 @@ body:not(.invoke-append-supported) .invoke-recall-btn[data-recall-mode="append_r display: none; } +/* The image-recall append option is irrelevant to the Video panel, whose + image placement route always understands append. */ +body.invoke-video-image-supported + .invoke-recall-controls[data-ref-target="video"] + .invoke-recall-btn[data-recall-mode="append_ref"] { + display: inline-flex; +} + .invoke-recall-btn { display: inline-flex; align-items: center; diff --git a/photomap/frontend/static/javascript/grid-view.js b/photomap/frontend/static/javascript/grid-view.js index bae7c0ab..69f342ca 100644 --- a/photomap/frontend/static/javascript/grid-view.js +++ b/photomap/frontend/static/javascript/grid-view.js @@ -6,6 +6,7 @@ import { updateCurrentImageScore, updateImageLabel, } from "./metadata-drawer.js"; +import { applyRefTarget } from "./invoke-ref-target.js"; import { fetchImageByIndex } from "./search.js"; import { slideState } from "./slide-state.js"; import { state } from "./state.js"; @@ -744,6 +745,7 @@ class GridViewManager { const recallControls = document.querySelector("#descriptionText .invoke-recall-controls"); if (recallControls) { recallContainer.appendChild(recallControls); + applyRefTarget(recallControls); } } diff --git a/photomap/frontend/static/javascript/invoke-capabilities.js b/photomap/frontend/static/javascript/invoke-capabilities.js index cebaba9c..92c17f78 100644 --- a/photomap/frontend/static/javascript/invoke-capabilities.js +++ b/photomap/frontend/static/javascript/invoke-capabilities.js @@ -5,12 +5,13 @@ // positively confirmed, so a backend without the recall router shows no // buttons at all, and one without the append option hides only // "Append to InvokeAI". The video group needs InvokeAI 7's video recall API -// and is gated on its own flag. +// and is gated on its own flag, as is the image/video choice for Send / +// Append Image, which needs InvokeAI 7's image placement route. import { fetchJson } from "./utils.js"; export async function refreshInvokeCapabilities({ refresh = false } = {}) { - let caps = { recall: false, append: false, video_recall: false }; + let caps = { recall: false, append: false, video_recall: false, video_image: false }; try { caps = await fetchJson(refresh ? "invokeai/capabilities?refresh=true" : "invokeai/capabilities"); } catch (err) { @@ -21,5 +22,6 @@ export async function refreshInvokeCapabilities({ refresh = false } = {}) { document.body.classList.toggle("invoke-recall-supported", caps.recall === true); document.body.classList.toggle("invoke-append-supported", caps.append === true); document.body.classList.toggle("invoke-video-recall-supported", caps.video_recall === true); + document.body.classList.toggle("invoke-video-image-supported", caps.video_image === true); return caps; } diff --git a/photomap/frontend/static/javascript/invoke-recall.js b/photomap/frontend/static/javascript/invoke-recall.js index 3517640f..7d1c8e5f 100644 --- a/photomap/frontend/static/javascript/invoke-recall.js +++ b/photomap/frontend/static/javascript/invoke-recall.js @@ -4,8 +4,10 @@ // request to the PhotoMap backend which in turn proxies a recall payload to // the configured InvokeAI backend. Videos get their own group (Initial Video, // Ref Video, and Recall / Remix for InvokeAI-generated videos), whose modes -// are prefixed ``video_``. +// are prefixed ``video_``. Send / Append Image go to InvokeAI's image or video +// generation tab, as chosen in the drawer's "as a reference for" pulldown. +import { refTargetFor } from "./invoke-ref-target.js"; import { state } from "./state.js"; import { fetchJson } from "./utils.js"; @@ -103,10 +105,10 @@ export async function sendRecall({ albumKey, index, includeSeed }) { } } -export async function sendUseRefImage({ albumKey, index, append = false }) { +export async function sendUseRefImage({ albumKey, index, append = false, target = "image" }) { try { return await fetchJson("invokeai/use_ref_image", { - json: { album_key: albumKey, index, append }, + json: { album_key: albumKey, index, append, target }, }); } catch (err) { throw _withDetailMessage(err); @@ -142,13 +144,18 @@ const FAILURE_MESSAGES = { video_ref: "Sending the reference video to InvokeAI failed", }; -async function dispatch(mode, albumKey, index) { +async function dispatch(mode, albumKey, index, button) { switch (mode) { case "use_ref": case "append_ref": // append_ref adds the image to InvokeAI's existing reference-image // list; use_ref replaces it. - return sendUseRefImage({ albumKey, index, append: mode === "append_ref" }); + return sendUseRefImage({ + albumKey, + index, + append: mode === "append_ref", + target: refTargetFor(button), + }); case "video_initial": return sendVideoMedia({ albumKey, index, target: "initial" }); case "video_ref": @@ -203,7 +210,7 @@ async function handleRecallClick(button) { button.disabled = true; try { - const result = await dispatch(mode, albumKey, parsed.index); + const result = await dispatch(mode, albumKey, parsed.index, button); if (result && result.success === false) { // InvokeAI answered, but recalled nothing (e.g. no model installed). showStatus(button, "error"); diff --git a/photomap/frontend/static/javascript/invoke-ref-target.js b/photomap/frontend/static/javascript/invoke-ref-target.js new file mode 100644 index 00000000..c3661bb4 --- /dev/null +++ b/photomap/frontend/static/javascript/invoke-ref-target.js @@ -0,0 +1,51 @@ +// invoke-ref-target.js +// The drawer's "as a reference for " +// pulldown, which picks the InvokeAI tab Send / Append Image feed. The choice +// is a persisted per-device setting (state.invokeRefTarget). Kept apart from +// invoke-recall.js so both drawer renderers (metadata-drawer.js and +// grid-view.js) can use it without pulling in the recall request code. + +import { setInvokeRefTarget, state } from "./state.js"; + +const REF_TARGETS = ["image", "video"]; + +function currentRefTarget() { + return REF_TARGETS.includes(state.invokeRefTarget) ? state.invokeRefTarget : "image"; +} + +// Show the remembered tab in a freshly rendered controls table. The drawer's +// HTML is rebuilt every slide, so both renderers call this after inserting it. +export function applyRefTarget(controls) { + const select = controls?.querySelector(".invoke-ref-target-select"); + if (!select) { + return; + } + const target = currentRefTarget(); + select.value = target; + controls.dataset.refTarget = target; +} + +// The tab a Send / Append click goes to. The pulldown is hidden on backends +// without InvokeAI 7's image placement route, so a remembered "video" must +// not apply there. +export function refTargetFor(button) { + if (!document.body.classList.contains("invoke-video-image-supported")) { + return "image"; + } + return button.closest(".invoke-recall-controls")?.dataset.refTarget === "video" ? "video" : "image"; +} + +document.addEventListener("change", (e) => { + const select = e.target.closest?.(".invoke-ref-target-select"); + if (!select || !REF_TARGETS.includes(select.value)) { + return; + } + setInvokeRefTarget(select.value); + const controls = select.closest(".invoke-recall-controls"); + if (controls) { + controls.dataset.refTarget = select.value; + } + // Hand the keyboard back to the slideshow shortcuts, which ignore a + // focused + + + + `; + return document.querySelector(".invoke-recall-controls"); +} + +function choose(controls, value) { + const select = controls.querySelector("select"); + select.value = value; + select.dispatchEvent(new Event("change", { bubbles: true })); +} + +describe("invoke-ref-target.js", () => { + beforeEach(() => { + mockState.invokeRefTarget = "image"; + setInvokeRefTarget.mockClear(); + document.body.classList.add("invoke-video-image-supported"); + }); + + afterEach(() => { + document.body.className = ""; + document.body.innerHTML = ""; + }); + + it("defaults to image generation", () => { + const controls = renderControls(); + applyRefTarget(controls); + + expect(controls.querySelector("select").value).toBe("image"); + expect(refTargetFor(controls.querySelector("button"))).toBe("image"); + }); + + it("remembers a choice across the drawer re-rendering for the next slide", () => { + choose(renderControls(), "video"); + expect(setInvokeRefTarget).toHaveBeenCalledWith("video"); + + const next = renderControls(); + applyRefTarget(next); + + expect(next.querySelector("select").value).toBe("video"); + expect(refTargetFor(next.querySelector("button"))).toBe("video"); + }); + + it("takes effect on the current slide immediately", () => { + const controls = renderControls(); + applyRefTarget(controls); + choose(controls, "video"); + + expect(refTargetFor(controls.querySelector("button"))).toBe("video"); + }); + + it("ignores a stored value it does not know", () => { + mockState.invokeRefTarget = "audio"; + const controls = renderControls(); + applyRefTarget(controls); + + expect(controls.querySelector("select").value).toBe("image"); + }); + + it("does nothing for a controls table without the pulldown (videos)", () => { + document.body.innerHTML = `
`; + const controls = document.querySelector(".invoke-recall-controls"); + + applyRefTarget(controls); + + expect(controls.dataset.refTarget).toBeUndefined(); + }); +});