From 1ce9c8bed86abe9c9375b585f1bf818da27ea804 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 3 Apr 2025 17:26:52 -0400 Subject: [PATCH] Added reference images to the generation enqueueImageGeneration now accepts additionalImages enqueueImageGeneration needs testing. Polling for jobs doesn't work, most api calls don't work and need integration testing. --- .../app/Classes/ApiManager/EngineApi.ts | 181 ++++++++++-------- .../reusable/UploadModalMedia/uploadImage.ts | 12 +- .../app/pages/PageEnigma/Editor/editor.ts | 11 +- .../PageEnigma/comps/PromptBox/PromptBox.tsx | 117 ++++++----- 4 files changed, 193 insertions(+), 128 deletions(-) diff --git a/frontend/editor3d/app/Classes/ApiManager/EngineApi.ts b/frontend/editor3d/app/Classes/ApiManager/EngineApi.ts index c8a5c9ad85..2ef494077a 100644 --- a/frontend/editor3d/app/Classes/ApiManager/EngineApi.ts +++ b/frontend/editor3d/app/Classes/ApiManager/EngineApi.ts @@ -47,130 +47,157 @@ export class EngineApi extends ApiManager { const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`; const formData = new FormData(); - formData.append("screenshot", screenshot); + formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec if (sceneMediaToken) { formData.append("scene_media_token", sceneMediaToken); } - return this.post< - FormData, - { success?: boolean; snapshot_media_token?: string; BadInput?: string } - >({ - endpoint, - body: formData, + const response = await fetch(endpoint, { + method: "POST", headers: { - "Content-Type": "multipart/form-data", + Accept: "application/json", }, - }) - .then((response) => { - return { - success: response.success ?? false, - data: response.snapshot_media_token, - errorMessage: response.BadInput, - }; - }) - .catch((err) => { - return { - success: false, - errorMessage: err.message, - }; - }); + credentials: "include", + body: formData, + }); + + const postResponse = await response.json(); + + console.log(postResponse); + + let result: { success: boolean; data?: string; errorMessage?: string }; + + if (postResponse.success) { + result = { + success: true, + data: postResponse.snapshot_media_token, + errorMessage: undefined, + }; + } else { + result = { + success: false, + errorMessage: postResponse.BadInput, + }; + } + + return result; } public async enqueueImageGeneration({ prompt, - sceneMediaToken, + snapshotMediaToken, additionalImages, }: { prompt: string; - sceneMediaToken: string; + snapshotMediaToken: string; additionalImages?: string[]; }): Promise> { const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`; const body = { prompt, - scene_media_token: sceneMediaToken, + snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token additional_images: additionalImages, }; - return this.post< + const postResponse = await this.post< { prompt: string; - scene_media_token: string; + snapshot_media_token: string; additional_images?: string[]; }, { success?: boolean; job_token?: string; BadInput?: string } >({ endpoint, body, - }) - .then((response) => { - return { - success: response.success ?? false, - data: response.job_token, - errorMessage: response.BadInput, - }; - }) - .catch((err) => { - return { - success: false, - errorMessage: err.message, - }; - }); + }); + + // Check if the response is successful + const isSuccess = postResponse.success ?? false; + + // Prepare the result object + const result = { + success: isSuccess, + data: isSuccess ? postResponse.job_token : undefined, + errorMessage: isSuccess ? undefined : postResponse.BadInput, + }; + + return result; } - public async pollJobSession(jobToken: string): Promise> { + public async pollJobSession(jobToken: string): Promise< + ApiResponse<{ + status: string; + result: { + generated_images?: string[]; + error?: string; + }; + }> + > { const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`; - return this.get<{ + const response = await this.get<{ success?: boolean; status?: string; - result?: any; + result?: { + generated_images?: string[]; + error?: string; + }; BadInput?: string; }>({ endpoint, - }) - .then((response) => { - return { - success: response.success ?? false, - data: response.result, - errorMessage: response.BadInput, - }; - }) - .catch((err) => { - return { - success: false, - errorMessage: err.message, - }; - }); + }); + + const success = response.success ?? false; + const status = response.status ?? ""; + const result = response.result ?? { + generated_images: [], + error: undefined, + }; + const errorMessage = response.BadInput; + + return { + success, + data: { status, result }, + errorMessage, + }; } - public async pollStudioSessionJobs( - jobToken: string, - ): Promise> { + public async pollStudioSessionJobs(jobToken: string): Promise< + ApiResponse<{ + status: string; + result: { + generated_images?: string[]; + error?: string; + }; + }> + > { const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`; - return this.get<{ + const response = await this.get<{ success?: boolean; status?: string; - result?: any; + result?: { + generated_images?: string[]; + error?: string; + }; BadInput?: string; }>({ endpoint, - }) - .then((response) => { - return { - success: response.success ?? false, - data: response.result, - errorMessage: response.BadInput, - }; - }) - .catch((err) => { - return { - success: false, - errorMessage: err.message, - }; - }); + }); + + const success = response.success ?? false; + const status = response.status ?? ""; + const result = response.result ?? { + generated_images: [], + error: undefined, + }; + const errorMessage = response.BadInput; + + return { + success, + data: { status, result }, + errorMessage, + }; } } diff --git a/frontend/editor3d/app/components/reusable/UploadModalMedia/uploadImage.ts b/frontend/editor3d/app/components/reusable/UploadModalMedia/uploadImage.ts index 9869b895b4..f2c5f0c43b 100644 --- a/frontend/editor3d/app/components/reusable/UploadModalMedia/uploadImage.ts +++ b/frontend/editor3d/app/components/reusable/UploadModalMedia/uploadImage.ts @@ -15,6 +15,8 @@ export const uploadImage = async ({ }) => { const mediaUploadApi = new MediaUploadApi(); + console.log("Uploading image:", assetFile); + progressCallback({ status: UploaderStates.uploadingImage }); const imageResponse = await mediaUploadApi.UploadImage({ @@ -24,7 +26,10 @@ export const uploadImage = async ({ maybe_title: "char_frame_" + title, }); + console.log("Image response:", imageResponse); + if (imageResponse == undefined) { + console.log("Error: Could not upload image!"); progressCallback({ status: UploaderStates.imageCreateError, errorMessage: "Could not upload image!", @@ -33,6 +38,7 @@ export const uploadImage = async ({ } if (!imageResponse.success || !imageResponse.data) { + console.log("Error:", imageResponse.errorMessage); progressCallback({ status: UploaderStates.imageCreateError, errorMessage: imageResponse.errorMessage, @@ -40,5 +46,9 @@ export const uploadImage = async ({ return; } - progressCallback({ status: UploaderStates.success, data: imageResponse.data }); + console.log("Upload successful:", imageResponse.data); + progressCallback({ + status: UploaderStates.success, + data: imageResponse.data, + }); }; diff --git a/frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts b/frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts index 5dfb6a9f86..0c95d76b4a 100644 --- a/frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts +++ b/frontend/editor3d/app/pages/PageEnigma/Editor/editor.ts @@ -657,14 +657,21 @@ class Editor { public snapShotOfCurrentFrame(shouldDownload: boolean = true) { const snapshot = this.renderer?.domElement.toDataURL("image/png"); + if (!snapshot) { + console.error("Error: Snapshot could not be created."); + return null; + } - if (snapshot) { + const base64Snapshot = snapshot.split(",")[1]; + + if (shouldDownload) { const link = document.createElement("a"); link.download = "scene-snapshot.png"; link.href = snapshot; link.click(); } - return snapshot; + + return base64Snapshot; } public async newScene(sceneTitleInput: string) { diff --git a/frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx b/frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx index ed8ad3981c..921e6ee0e4 100644 --- a/frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx +++ b/frontend/editor3d/app/pages/PageEnigma/comps/PromptBox/PromptBox.tsx @@ -2,7 +2,7 @@ import { useState, useRef, useEffect } from "react"; import { useSignals } from "@preact/signals-react/runtime"; import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; import { useContext } from "react"; - +import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage"; import { faPlus, faCamera, @@ -34,9 +34,11 @@ interface ReferenceImage { id: string; url: string; file: File; + mediaToken: string; } import { EngineApi } from "~/Classes/ApiManager/EngineApi"; import { scene } from "~/signals/scene"; +import { UploaderStates } from "~/enums/UploaderStates"; export const PromptBox = () => { useSignals(); @@ -169,14 +171,25 @@ export const PromptBox = () => { Array.from(files).forEach((file) => { const reader = new FileReader(); reader.onloadend = () => { - setReferenceImages((prev) => [ - ...prev, - { - id: Math.random().toString(36).substring(7), - url: reader.result as string, - file, + uploadImage({ + title: `reference-image-${Math.random().toString(36).substring(2, 15)}`, + assetFile: file, + progressCallback: (newState) => { + console.debug("Upload progress:", newState.data); + if (newState.status === UploaderStates.success && newState.data) { + const referenceImage: ReferenceImage = { + id: Math.random().toString(36).substring(7), + url: reader.result as string, + file, + mediaToken: newState.data || "", + }; + setReferenceImages((prev) => [...prev, referenceImage]); + console.log("Reference image added:", referenceImage); + } else { + console.error("No data from uploadImage"); + } }, - ]); + }); }; reader.readAsDataURL(file); }); @@ -226,28 +239,34 @@ export const PromptBox = () => { if (editorEngine) { editorEngine.positive_prompt = prompt; - } - const engineApi = new EngineApi(); + const engineApi = new EngineApi(); + const snapshot = editorEngine.snapShotOfCurrentFrame(false); - engineApi.enqueueImageGeneration({ - prompt: prompt, - sceneMediaToken: scene.value.token || "", - additionalImages: [], - }); + const snapshotResult = await engineApi.uploadSceneSnapshot({ + screenshot: snapshot || "", + sceneMediaToken: "", + }); - setisEnqueueing(true); - try { - // Here we would pass both the prompt and reference images to the generation - console.log( - "Enqueuing with prompt:", - prompt, - "and reference images:", - referenceImages, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } finally { - setisEnqueueing(false); + await engineApi.enqueueImageGeneration({ + prompt: prompt, + snapshotMediaToken: snapshotResult.data || "", + additionalImages: referenceImages.map((image) => image.mediaToken), + }); + + setisEnqueueing(true); + try { + // Here we would pass both the prompt and reference images to the generation + console.log( + "Enqueuing with prompt:", + prompt, + "and reference images:", + referenceImages, + ); + await new Promise((resolve) => setTimeout(resolve, 2000)); + } finally { + setisEnqueueing(false); + } } }; @@ -257,37 +276,37 @@ export const PromptBox = () => { handleEnqueue(); } }; - -// Get the current aspect ratio icon based on the cameraAspectRatio signal -const getCurrentAspectRatioIcon = () => { - switch (cameraAspectRatio.value) { - case CameraAspectRatio.HORIZONTAL_16_9: - return faRectangleWide; - case CameraAspectRatio.HORIZONTAL_3_2: - return faRectangle; - case CameraAspectRatio.VERTICAL_2_3: - return faRectangleVertical; - case CameraAspectRatio.VERTICAL_9_16: - return faRectangleVertical; - case CameraAspectRatio.SQUARE_1_1: - return faSquare; - default: - return faRectangleWide; + + // Get the current aspect ratio icon based on the cameraAspectRatio signal + const getCurrentAspectRatioIcon = () => { + switch (cameraAspectRatio.value) { + case CameraAspectRatio.HORIZONTAL_16_9: + return faRectangleWide; + case CameraAspectRatio.HORIZONTAL_3_2: + return faRectangle; + case CameraAspectRatio.VERTICAL_2_3: + return faRectangleVertical; + case CameraAspectRatio.VERTICAL_9_16: + return faRectangleVertical; + case CameraAspectRatio.SQUARE_1_1: + return faSquare; + default: + return faRectangleWide; } }; - const handleSaveFrame = () => { + const handleSaveFrame = async () => { if (editorEngine) { const snapshot = editorEngine.snapShotOfCurrentFrame(false); - const engineApi = new EngineApi(); - - engineApi.uploadSceneSnapshot({ + const result = await engineApi.uploadSceneSnapshot({ screenshot: snapshot || "", - sceneMediaToken: scene.value.token || "", + sceneMediaToken: "", }); + console.log(result); } }; + const handleDownloadFrame = () => { if (editorEngine) { editorEngine.snapShotOfCurrentFrame(true); @@ -396,6 +415,7 @@ const getCurrentAspectRatioIcon = () => { className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90" variant="secondary" icon={faDownload} + onClick={handleDownloadFrame} > Download frame @@ -403,6 +423,7 @@ const getCurrentAspectRatioIcon = () => { className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90" variant="secondary" icon={faSave} + onClick={handleSaveFrame} > Save frame