diff --git a/frontend/apps/editor3d/app/src/pages/PageDraw/utilities/EncodeImageBitmapToArray.ts b/frontend/apps/editor3d/app/src/pages/PageDraw/utilities/EncodeImageBitmapToArray.ts new file mode 100644 index 0000000000..aedc60dc97 --- /dev/null +++ b/frontend/apps/editor3d/app/src/pages/PageDraw/utilities/EncodeImageBitmapToArray.ts @@ -0,0 +1,32 @@ + +export const EncodeImageBitmapToArray = async (imageBitmap: ImageBitmap): Promise => { + + + const canvas = new OffscreenCanvas(imageBitmap.width, imageBitmap.height); + const ctx = canvas.getContext('2d'); + ctx.drawImage(imageBitmap, 0, 0); + + const imageData = ctx.getImageData(0, 0, imageBitmap.width, imageBitmap.height); + const clampedArray = imageData.data; // Uint8ClampedArray + return new Uint8Array(clampedArray.buffer); + + /*// Create a temporary canvas + const canvas = document.createElement("canvas"); + canvas.width = imageBitmap.width; + canvas.height = imageBitmap.height; + + // Draw the ImageBitmap onto the canvas + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Failed to get 2D context"); + + ctx.drawImage(imageBitmap, 0, 0); + + // Convert to base64 + const base64String = canvas.toDataURL("image/png"); + + // Clean up + canvas.remove(); + + // Remove the data:image/png;base64, prefix if you want just the base64 string + return base64String.split(",")[1];*/ +} diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts new file mode 100644 index 0000000000..002059b3ed --- /dev/null +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts @@ -0,0 +1,95 @@ +import { invoke } from "@tauri-apps/api/core"; +import { CommandResult } from "../common/CommandStatus"; +import { ModelInfo } from "@storyteller/model-list"; + +export interface EnqueueImageInpaintRequest { + // The model to use. + model?: ModelInfo | EnqueueImageInpaintModel; + + // The image we're editing. + image_media_token?: string; + + // The mask to focus the edit (raw bytes). + mask_image_raw_bytes?: Uint8Array; + + // The user's image generation prompt. + prompt?: string; + + // Number of images to generate. + image_count?: number; +} + + +// Shape of request sent to Tauri +// (We do some transformations from the public-facing request object.) +interface RawEnqueueImageInpaintRequest { + model?: EnqueueImageInpaintModel | string; + image_media_token?: string; + mask_image_raw_bytes?: Uint8Array; + prompt?: string; + image_count?: number; +} + +export enum EnqueueImageInpaintErrorType { + ModelNotSpecified = "model_not_specified", + NoProviderAvailable = "no_provider_available", + BadRequest = "bad_request", + ServerError = "server_error", + TooManyConcurrentTasks = "too_many_concurrent_tasks", + SoraLoginRequired = "sora_login_required", + SoraUsernameNotYetCreated = "sora_username_not_yet_created", + SoraIsHavingProblems = "sora_is_having_problems", +} + +export enum EnqueueImageInpaintModel { + FluxPro1 = "flux_pro_1", +} + +export interface EnqueueImageInpaintError extends CommandResult { + error_type: EnqueueImageInpaintErrorType; + error_message?: string; +} + +export interface EnqueueImageInpaintPayload { +} + +export interface EnqueueImageInpaintSuccess extends CommandResult { + payload: EnqueueImageInpaintPayload; +} + +export type EnqueueImageInpaintResult = EnqueueImageInpaintSuccess | EnqueueImageInpaintError; + +export const EnqueueImageInpaint = async (request: EnqueueImageInpaintRequest) : Promise => { + let modelName = undefined; + + if (!!request.model) { + if (typeof request.model === "string") { + modelName = request.model; + } else { + modelName = request.model.tauri_id; + } + } + + let mutableRequest : RawEnqueueImageInpaintRequest = { + model: modelName, + prompt: request.prompt, + }; + + if (!!request.image_media_token) { + mutableRequest.image_media_token = request.image_media_token; + } + + if (!!request.mask_image_raw_bytes) { + mutableRequest.mask_image_raw_bytes = request.mask_image_raw_bytes; + } + + if (!!request.image_count) { + mutableRequest.image_count = request.image_count; + } + + const result = await invoke("enqueue_image_inpaint_command", { + request: mutableRequest, + }); + + return (result as EnqueueImageInpaintResult); +}