From e27ebc79617570da1d09dc134c59114fb5039ddd Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Tue, 8 Apr 2025 18:56:57 -0400 Subject: [PATCH] properly wired up the image generation api (#488) Co-authored-by: Michael Chung --- frontend/editor2d/src/KonvaApp/Api.ts | 578 ++++++++++++++++++ .../editor2d/src/components/PromptBox.tsx | 98 +-- 2 files changed, 637 insertions(+), 39 deletions(-) create mode 100644 frontend/editor2d/src/KonvaApp/Api.ts diff --git a/frontend/editor2d/src/KonvaApp/Api.ts b/frontend/editor2d/src/KonvaApp/Api.ts new file mode 100644 index 0000000000..90ddcd1f75 --- /dev/null +++ b/frontend/editor2d/src/KonvaApp/Api.ts @@ -0,0 +1,578 @@ +type NonNullableObject = NonNullable; + +export interface ApiResponse { + success: boolean; + errorMessage?: string; + data?: T; + pagination?: P; +} + +export class ApiManager { + ApiTargets: Record = {}; + + constructor() { + // look at the .env file + this.ApiTargets = { + BaseApi: "https://api.storyteller.ai", + CdnApi: "https://cdn.storyteller.ai", + }; + } + + public async fetch( + endpoint: string, + { + method, + query, + body, + }: { + method: string; + query?: Record; + body?: B; + }, + ): Promise { + const queryInString = + query && + Object.entries(query).reduce( + (allOptions, [key, value]) => { + if (!value) { + return allOptions; + } + allOptions[key] = value.toString(); + return allOptions; + }, + {} as Record, + ); + + const endpointWithQueries = queryInString + ? endpoint + "?" + new URLSearchParams(queryInString) + : endpoint; + + const bodyInString = JSON.stringify(body); + + const response = await fetch(endpointWithQueries, { + method, + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + credentials: "include", + body: bodyInString, + }); + return response.json(); + } + + public async fetchMultipartFormData( + endpoint: string, + { + method, + body, + }: { + method: string; + body: FormData; + }, + ): Promise { + const response = await fetch(endpoint, { + method, + headers: { + Accept: "application/json", + }, + credentials: "include", + body: body, + }); + return response.json(); + } + + protected get({ + endpoint, + query, + }: { + endpoint: string; + query?: Record; + }): Promise { + return this.fetch(endpoint, { method: "GET", query }); + } + + protected post({ + endpoint, + query, + body, + }: { + endpoint: string; + query?: Record; + body?: B; + }): Promise { + return this.fetch(endpoint, { + method: "POST", + query, + body, + }); + } + + protected delete({ + endpoint, + query, + body, + }: { + endpoint: string; + query?: Record; + body?: B; + }): Promise { + return this.fetch(endpoint, { + method: "DELETE", + query, + body, + }); + } + + protected async postForm({ + endpoint, + formRecord, + uuid, + blob, + blobFileName, + }: { + endpoint: string; + formRecord: Record; + uuid: string; + blob?: Blob | File; + blobFileName?: string; + }): Promise { + const formData = new FormData(); + formData.append("uuid_idempotency_token", uuid); + Object.entries(formRecord).forEach(([key, value]) => { + formData.append(key, value); + }); + if (blob && blobFileName) { + formData.append("file", blob, blobFileName); + } else if (blob) { + formData.append("file", blob); + } + + console.log(`HELLO${formData}`); + return this.fetchMultipartFormData(endpoint, { + method: "POST", + headers: { + Accept: "application/json", + }, + credentials: "include", + body: formData, + }); + } + + protected camelToSnakeCase(str: string) { + return str.replace(/([a-z0])([A-Z])/g, "$1_$2").toLowerCase(); + } + + protected parseQueryValues( + params: Record, + ): Record { + return Object.entries(params).reduce( + (allParams, [key, value]) => { + if (!value) { + return allParams; + } + const snakeKey = this.camelToSnakeCase(key); + if (Array.isArray(value)) { + return { ...allParams, [snakeKey]: value.join(",") }; + } + return { ...allParams, [snakeKey]: value.toString() }; + }, + {} as Record, + ); + } + + protected parseBodyValues( + params: NonNullableObject, + ): B { + return Object.entries(params).reduce((allParams, [key, value]) => { + if (!value) { + return allParams; + } + const snakeKey = this.camelToSnakeCase(key); + if (Array.isArray(value)) { + return { ...allParams, [snakeKey]: value }; + } + return { ...allParams, [snakeKey]: value }; + }, {} as B); + } +} + +export class Api extends ApiManager { + public async ConvertTbxToGltf({ + mediaFileToken, + uuidIdempotencyToken, + }: { + mediaFileToken: string; + uuidIdempotencyToken: string; + }): Promise> { + const endpoint = `${this.ApiTargets.BaseApi}/v1/conversion/enqueue_fbx_to_gltf`; + + const body = { + media_file_token: mediaFileToken, + uuid_idempotency_token: uuidIdempotencyToken, + }; + + return this.post< + { media_file_token: string; uuid_idempotency_token: string }, + { success?: boolean; inference_job_token?: string; BadInput?: string } + >({ + endpoint, + body: body, + }) + .then((response) => { + return { + success: response.success ?? false, + data: response.inference_job_token, + errorMessage: response.BadInput, + }; + }) + .catch((err) => { + return { + success: false, + errorMessage: err.message, + }; + }); + } + + public async uploadSceneSnapshot({ + screenshot, + sceneMediaToken, + }: { + screenshot: File; // base64 encoded PNG + sceneMediaToken?: string; + }): Promise> { + const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`; + const formData = new FormData(); + formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec + if (sceneMediaToken) { + formData.append("scene_media_token", sceneMediaToken); + } + + // for now ... + const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID + formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Accept: "application/json", + }, + credentials: "include", + body: formData, + }); + + const postResponse = await response.json(); + + console.log(postResponse); + + let result: { success: boolean; data?: string; errorMessage?: string }; + + if (postResponse.success) { + result = { + success: true, + data: postResponse.snapshot_media_token, + errorMessage: undefined, + }; + } else { + result = { + success: false, + errorMessage: postResponse.BadInput, + }; + } + + return result; + } + + public async enqueueImageGeneration({ + prompt, + snapshotMediaToken, + additionalImages, + }: { + prompt: string; + snapshotMediaToken: string; + additionalImages?: string[]; + }): Promise> { + const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`; + // for now ... + const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID + const body = { + uuid_idempotency_token: uuidIdempotencyToken, + prompt, + snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token + additional_images: additionalImages, + }; + + const postResponse = await this.post< + { + uuid_idempotency_token: string; + prompt: string; + snapshot_media_token: string; + additional_images?: string[]; + }, + { success?: boolean; job_token?: string; BadInput?: string } + >({ + endpoint, + body, + }); + + // Check if the response is successful + const isSuccess = postResponse.success ?? false; + + // Prepare the result object + const result = { + success: isSuccess, + data: isSuccess ? postResponse.job_token : undefined, + errorMessage: isSuccess ? undefined : postResponse.BadInput, + }; + + return result; + } + + public async pollJobSession(jobToken: string): Promise< + ApiResponse<{ + status: string; + result: { + generated_images?: string[]; + error?: string; + }; + }> + > { + const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`; + + const response = await this.get<{ + success?: boolean; + status?: string; + result?: { + generated_images?: string[]; + error?: string; + }; + BadInput?: string; + }>({ + endpoint, + }); + + const success = response.success ?? false; + const status = response.status ?? ""; + const result = response.result ?? { + generated_images: [], + error: undefined, + }; + const errorMessage = response.BadInput; + + return { + success, + data: { status, result }, + errorMessage, + }; + } + + public async pollStudioSessionJobs(jobToken: string): Promise< + ApiResponse<{ + status: string; + result: { + generated_images?: string[]; + error?: string; + }; + }> + > { + const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`; + + const response = await this.get<{ + success?: boolean; + status?: string; + result?: { + generated_images?: string[]; + error?: string; + }; + BadInput?: string; + }>({ + endpoint, + }); + + const success = response.success ?? false; + const status = response.status ?? ""; + const result = response.result ?? { + generated_images: [], + error: undefined, + }; + const errorMessage = response.BadInput; + + return { + success, + data: { status, result }, + errorMessage, + }; + } +} + +export class EngineApi extends ApiManager { + public async uploadSceneSnapshot({ + screenshot, + sceneMediaToken, + }: { + screenshot: File; // base64 encoded PNG + sceneMediaToken?: string; + }): Promise> { + const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/scene_snapshot`; + const formData = new FormData(); + formData.append("snapshot", screenshot); // Changed from "screenshot" to "snapshot" to match API spec + if (sceneMediaToken) { + formData.append("scene_media_token", sceneMediaToken); + } + + // for now ... + const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID + formData.append("uuid_idempotency_token", uuidIdempotencyToken); // Added uuid_idempotency_token + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Accept: "application/json", + }, + credentials: "include", + body: formData, + }); + + const postResponse = await response.json(); + + console.log(postResponse); + + let result: { success: boolean; data?: string; errorMessage?: string }; + + if (postResponse.success) { + result = { + success: true, + data: postResponse.snapshot_media_token, + errorMessage: undefined, + }; + } else { + result = { + success: false, + errorMessage: postResponse.BadInput, + }; + } + + return result; + } + + public async enqueueImageGeneration({ + prompt, + snapshotMediaToken, + additionalImages, + }: { + prompt: string; + snapshotMediaToken: string; + additionalImages?: string[]; + }): Promise> { + const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/prompt`; + // for now ... + const uuidIdempotencyToken = crypto.randomUUID(); // Generate a new UUID + const body = { + uuid_idempotency_token: uuidIdempotencyToken, + prompt, + snapshot_media_token: snapshotMediaToken, // Changed from scene_media_token to snapshot_media_token + additional_images: additionalImages, + }; + + const postResponse = await this.post< + { + uuid_idempotency_token: string; + prompt: string; + snapshot_media_token: string; + additional_images?: string[]; + }, + { success?: boolean; job_token?: string; BadInput?: string } + >({ + endpoint, + body, + }); + + // Check if the response is successful + const isSuccess = postResponse.success ?? false; + + // Prepare the result object + const result = { + success: isSuccess, + data: isSuccess ? postResponse.job_token : undefined, + errorMessage: isSuccess ? undefined : postResponse.BadInput, + }; + + return result; + } + + public async pollJobSession(jobToken: string): Promise< + ApiResponse<{ + status: string; + result: { + generated_images?: string[]; + error?: string; + }; + }> + > { + const endpoint = `${this.ApiTargets.BaseApi}/v1/jobs/session/${jobToken}`; + + const response = await this.get<{ + success?: boolean; + status?: string; + result?: { + generated_images?: string[]; + error?: string; + }; + BadInput?: string; + }>({ + endpoint, + }); + + const success = response.success ?? false; + const status = response.status ?? ""; + const result = response.result ?? { + generated_images: [], + error: undefined, + }; + const errorMessage = response.BadInput; + + return { + success, + data: { status, result }, + errorMessage, + }; + } + + public async pollStudioSessionJobs(jobToken: string): Promise< + ApiResponse<{ + status: string; + result: { + generated_images?: string[]; + error?: string; + }; + }> + > { + const endpoint = `${this.ApiTargets.BaseApi}/v1/image_studio/session_jobs/${jobToken}`; + + const response = await this.get<{ + success?: boolean; + status?: string; + result?: { + generated_images?: string[]; + error?: string; + }; + BadInput?: string; + }>({ + endpoint, + }); + + const success = response.success ?? false; + const status = response.status ?? ""; + const result = response.result ?? { + generated_images: [], + error: undefined, + }; + const errorMessage = response.BadInput; + + return { + success, + data: { status, result }, + errorMessage, + }; + } +} diff --git a/frontend/editor2d/src/components/PromptBox.tsx b/frontend/editor2d/src/components/PromptBox.tsx index 732292edca..1f97b669bb 100644 --- a/frontend/editor2d/src/components/PromptBox.tsx +++ b/frontend/editor2d/src/components/PromptBox.tsx @@ -32,7 +32,7 @@ import { getCanvasRenderBitmap } from "~/signal/canvasRenderBitmap"; import { EncodeImageBitmapToBase64 } from "~/utilities/EncodeImageBitmapToBase64"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { PopoverItem } from "~/components/reusable/Popover"; - +import { Api, ApiManager } from "~/KonvaApp/Api"; interface ReferenceImage { id: string; url: string; @@ -42,8 +42,10 @@ interface ReferenceImage { export const PromptBox = () => { useSignals(); - //const { lastRenderedBitmap } = useCanvasSignal(); + const isDesktopApp = window.navigator.userAgent.includes("Tauri"); + console.log("Is this a desktop app?", isDesktopApp); + //const { lastRenderedBitmap } = useCanvasSignal(); const [prompt, setPrompt] = useState(""); const [isEnqueueing, setisEnqueueing] = useState(false); const [useSystemPrompt, setUseSystemPrompt] = useState(true); @@ -188,20 +190,65 @@ export const PromptBox = () => { referenceImages, ); - let image = getCanvasRenderBitmap(); - - if (image === undefined) { - return; + if (isDesktopApp) { + let image = getCanvasRenderBitmap(); + if (image === undefined) { + return; + } + const base64Bitmap = await EncodeImageBitmapToBase64(image); + const generateResponse = await invoke("image_generation_command", { + image: base64Bitmap, + prompt: prompt, + }); + await new Promise((resolve) => setTimeout(resolve, 2000)); } - const base64Bitmap = await EncodeImageBitmapToBase64(image); + if (!isDesktopApp) { + const api = new Api(); + let image = getCanvasRenderBitmap(); + if (image === undefined) { + console.error( + "Error: Unable to generate image. Please check the input and try again.", + ); + return; + } + const base64Bitmap = await EncodeImageBitmapToBase64(image); + const byteString = atob(base64Bitmap); + const mimeString = "image/png"; + const ab = new ArrayBuffer(byteString.length); + const ia = new Uint8Array(ab); + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + const uuid = crypto.randomUUID(); // Generate a new UUID + const file = new File([ab], `${uuid}.png`, { type: mimeString }); - const generateResponse = await invoke("image_generation_command", { - image: base64Bitmap, - prompt: prompt, - }); + const snapshotMediaToken = await api.uploadSceneSnapshot({ + screenshot: file, + }); - await new Promise((resolve) => setTimeout(resolve, 2000)); + if (snapshotMediaToken.data === undefined) { + console.error("Error: Unable to upload scene snapshot."); + return; + } + + console.log("Snapshot media token:", snapshotMediaToken.data); + + const response = await api.enqueueImageGeneration({ + prompt: prompt, + snapshotMediaToken: snapshotMediaToken.data, + additionalImages: referenceImages.map((image) => image.mediaToken), + }); + + console.log("Response:", response); + + if (response.success === true) { + console.error("Successfully enqueued image generation."); + return; + } else { + console.log("Image generation Not Successfull."); + } + } } finally { setisEnqueueing(false); } @@ -214,10 +261,6 @@ export const PromptBox = () => { return iconElement.props.icon; }; - // const handleSaveFrame = async () => {}; - - // const handleDownloadFrame = () => {}; - const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); @@ -360,29 +403,6 @@ export const PromptBox = () => {
- {/* - */}