From 67d1e7320865ca5bd623808baa517511118d26bf Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Wed, 13 May 2026 13:53:03 -0700 Subject: [PATCH] promptbox: route webapp Generate through OmniGenApi (fix 404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PromptBox3D's handleWebEnqueue was POSTing to /v1/image_studio/prompt via PromptsApi.enqueueImageGeneration, but that endpoint does not exist on the Rust backend — image_studio_routes.rs only registers /scene_snapshot and /update_job_status under /v1/image_studio (the remaining routes live under legacy_routes/). The webapp's /create-image and other generation flows already use OmniGenApi.generateImage hitting /v1/omni_gen/image/generate; this commit aligns the 3D editor with that modern path. - Snapshot upload still goes through PromptsApi.uploadSceneSnapshot (that endpoint is the one image_studio route still in use). - The returned snapshot media token is prepended to image_media_tokens ahead of the user's reference images, matching how the Tauri path treats the scene snapshot as the primary input. - aspect_ratio uses commonAspectRatio for models that support the new shape, falling back to getCurrentAspectRatio() otherwise. - On success: toast "Image added to queue" plus the same __storeTaskEnqueueMeta telemetry the Tauri path fires. - Failure path routes through the existing handleError modal. - Drops a dead 2-second setTimeout that delayed the success toast. Tauri path unchanged — IsDesktopApp() still gates handleTauriEnqueue, which goes through Tauri IPC and bypasses storyteller-web entirely. --- .../promptbox/src/lib/PromptBox3D.tsx | 118 +++++++++++------- 1 file changed, 76 insertions(+), 42 deletions(-) diff --git a/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx b/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx index 1d5950261b..94ab992be3 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx @@ -26,7 +26,8 @@ import { FocalLengthDragging, UploadImageArgs, } from "@storyteller/common"; -import { PromptsApi } from "@storyteller/api"; +import { OmniGenApi, PromptsApi } from "@storyteller/api"; +import type { OmniGenImageRequest } from "@storyteller/api"; // import { SoundRegistry } from "@storyteller/soundboard"; import { toast } from "@storyteller/ui-toaster"; import { EngineApi } from "@storyteller/api"; @@ -308,55 +309,88 @@ export const PromptBox3D = ({ const handleWebEnqueue = async () => { if (!prompt.trim()) return; - setIsEnqueueing(true); + // OmniGen's request shape requires a model id. The Tauri path can + // fall back on `tauriId`, but the web path goes straight to the + // omni-gen backend which keys off the canonical `model` / `id` + // string. Bail loudly if nothing is selected. + if (!selectedImageModel?.id) { + handleError("No model selected"); + return; + } + setIsEnqueueing(true); setEnginePrompt(prompt); - //const engineApi = new EngineApi(); - const promptsApi = new PromptsApi(); - - if (snapshotCurrentFrame) { - const snapshot = snapshotCurrentFrame(false); - - if (snapshot) { - const snapshotResult = await promptsApi.uploadSceneSnapshot({ - screenshot: snapshot.file, - sceneMediaToken: "", - }); - - console.log("(web 3d) useSystemPrompt", useSystemPrompt); - - const response = await promptsApi.enqueueImageGeneration({ - disableSystemPrompt: !useSystemPrompt, - prompt: prompt, - snapshotMediaToken: snapshotResult.data || "", - additionalImages: referenceImages.map((image) => image.mediaToken), - }); - - console.log("response", response); - - if (response.errorMessage) { - handleError(response.errorMessage); - setIsEnqueueing(false); - return; + try { + // Snapshot upload still goes through the legacy /v1/image_studio/ + // scene_snapshot endpoint — that's the one image_studio route the + // Rust backend keeps registered, and it returns the media token we + // pass through to OmniGen as the canvas image. + let snapshotToken: string | undefined; + if (snapshotCurrentFrame) { + const snapshot = snapshotCurrentFrame(false); + if (snapshot) { + const snapshotResult = await new PromptsApi().uploadSceneSnapshot({ + screenshot: snapshot.file, + sceneMediaToken: "", + }); + if (!snapshotResult.success || !snapshotResult.data) { + handleError( + snapshotResult.errorMessage || "Failed to upload scene snapshot", + ); + return; + } + snapshotToken = snapshotResult.data; } } - try { - // Here we would pass both the prompt and reference images to the generation - console.log( - "(3D.2) Enqueuing with prompt:", - prompt, - "and reference images:", - referenceImages, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } finally { - setIsEnqueueing(false); - toast.success("Image added to queue"); + const referenceTokens = referenceImages + .map((image) => image.mediaToken) + .filter((t): t is string => !!t && t.length > 0); + const imageMediaTokens = [ + ...(snapshotToken ? [snapshotToken] : []), + ...referenceTokens, + ]; + + const aspectRatio = + selectedImageModel.supportsNewAspectRatio() && commonAspectRatio + ? commonAspectRatio + : getCurrentAspectRatio(); + + const request: OmniGenImageRequest = { + model: selectedImageModel.id, + prompt: prompt, + idempotency_token: crypto.randomUUID(), + aspect_ratio: aspectRatio ?? null, + resolution: getCurrentResolution() ?? null, + image_batch_count: 1, + image_media_tokens: + imageMediaTokens.length > 0 ? imageMediaTokens : null, + }; + + const response = await new OmniGenApi().generateImage(request); + if (!response.success || !response.inference_job_token) { + handleError("Generation failed"); + return; } + + window.__storeTaskEnqueueMeta?.({ + prompt, + refImageUrls: referenceImages + ?.map((image) => image.url) + .filter(Boolean), + modelType: selectedImageModel.id, + timestamp: Date.now(), + }); + + toast.success("Image added to queue"); + } catch (err) { + handleError( + err instanceof Error ? err.message : "Generation request failed", + ); + } finally { + setIsEnqueueing(false); } - setIsEnqueueing(false); }; // Helper to show Sora login reminder and wait for login