diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx index ea66b0e8b9..a6d99016a9 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx @@ -1,5 +1,5 @@ -import { useState, useRef, useEffect } from "react"; -import { INPAINT_LAYER_ID, PaintSurface } from "./PaintSurface"; +import { useState, useRef, useEffect, useCallback } from "react"; +import { DRAW_LAYER_ID, INPAINT_LAYER_ID, PaintSurface } from "./PaintSurface"; import "./App.css"; import PromptEditor from "./PromptEditor/PromptEditor"; import SideToolbar from "./components/ui/SideToolbar"; @@ -8,25 +8,26 @@ import { useUndoRedoHotkeys } from "./hooks/useUndoRedoHotkeys"; import { useDeleteHotkeys } from "./hooks/useDeleteHotkeys"; import { useCopyPasteHotkeys } from "./hooks/useCopyPasteHotkeys"; import Konva from "konva"; -import { setCanvasRenderBitmap } from "../../signals/canvasRenderBitmap"; -import { captureStageImageBitmap } from "./hooks/useUpdateSnapshot"; +import { captureStageEditsBitmap } from "./hooks/useUpdateSnapshot"; import { ContextMenuContainer } from "./components/ui/ContextMenu"; import { ImageModel } from "@storyteller/model-list"; import { CANVAS_2D_PAGE_MODEL_LIST, ClassyModelSelector, ModelPage, - //ProviderSelector, - //PROVIDER_LOOKUP_BY_PAGE, useSelectedImageModel, useSelectedProviderForModel, } from "@storyteller/ui-model-selector"; -import { useCanvasBgRemovedEvent } from "@storyteller/tauri-api"; +import { EnqueueEditImage, EnqueueEditImageRequest, EnqueueEditImageResolution, EnqueueEditImageSize, EnqueueImageInpaint, EnqueueImageInpaintRequest, useCanvasBgRemovedEvent } from "@storyteller/tauri-api"; import { HelpMenuButton } from "@storyteller/ui-help-menu"; import { GenerationProvider } from "@storyteller/api-enums"; import { HistoryStack, ImageBundle } from "../PageEdit/HistoryStack"; import { BaseImageSelector, BaseSelectorImage } from "../PageEdit/BaseImageSelector"; import { normalizeCanvas } from "~/Helpers/CanvasHelpers"; +import { EncodeImageBitmapToBase64 } from "./utilities/EncodeImageBitmapToBase64"; +import { RefImage, usePrompt2DStore } from "@storyteller/ui-promptbox"; +import { PromptsApi } from "@storyteller/api"; +import toast from "react-hot-toast"; const PAGE_ID: ModelPage = ModelPage.Canvas2D; @@ -62,6 +63,13 @@ const PageDraw = () => { const stageRef = useRef({} as Konva.Stage); const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({}); const store = useSceneStore(); + const promptStoreProvider = usePrompt2DStore; + const generationCount = promptStoreProvider((state) => state.generationCount); + const setGenerationCount = promptStoreProvider((state) => state.setGenerationCount); + const useSystemPrompt = promptStoreProvider((state) => state.useSystemPrompt); + const referenceImages = promptStoreProvider((state) => state.referenceImages); + const prompt = promptStoreProvider((state) => state.prompt); + const baseImageKonvaRef = useRef({} as Konva.Image); const baseImageUrl = store.baseImageInfo?.url; const [pendingGenerations, setPendingGenerations] = useState< @@ -122,7 +130,7 @@ const PageDraw = () => { throw new Error("Stage or left panel or base image ref is not available"); } - const layer = stageRef.current.getLayers().find((l) => l.name() === INPAINT_LAYER_ID)!; + const layer = stageRef.current.getLayers().find((l) => l.id() === INPAINT_LAYER_ID)!; // Get the canvas area that's covered by the image/rectangle const rect = baseImageKonvaRef.current; @@ -238,23 +246,279 @@ const PageDraw = () => { } }; - const onEnqueuedPressed = async () => { - const { width, height } = store.getAspectRatioDimensions(); + const handleTauriEnqueue = async (image: ImageBitmap, aspectRatio: EnqueueEditImageSize | undefined, resolution: EnqueueEditImageResolution | undefined, subscriberId: string) => { + if (image === undefined) { + console.log("image is undefined"); + return; + } - // takes snap shot and then a global variable in the engine will invoke the inference. - const image = await captureStageImageBitmap( - stageRef, - transformerRefs, + const api = new PromptsApi(); + const base64Bitmap = await EncodeImageBitmapToBase64(image); + + const byteString = atob(base64Bitmap); + const mimeString = "image/png"; + + const ab = new ArrayBuffer(byteString.length); + const ia = new Uint8Array(ab); + + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + + const uuid = crypto.randomUUID(); // Generate a new UUID + const file = new File([ab], `${uuid}.png`, { type: mimeString }); + + const snapshotMediaToken = await api.uploadSceneSnapshot({ + screenshot: file, + }); + + if (snapshotMediaToken.data === undefined) { + toast.error("Error: Unable to upload scene snapshot Please try again."); + return; + } + + console.log("useSystemPrompt", useSystemPrompt); + console.log("Snapshot media token:", snapshotMediaToken.data); + + const request: EnqueueEditImageRequest = { + model: selectedImageModel, + scene_image_media_token: snapshotMediaToken.data!, + image_media_tokens: referenceImages + .map((image) => image.mediaToken) + .filter((t) => t.length > 0), + disable_system_prompt: !useSystemPrompt, + prompt: prompt, + image_count: generationCount, + aspect_ratio: aspectRatio, + image_resolution: resolution, + frontend_caller: "image_editor", + frontend_subscriber_id: subscriberId, + }; + + if (selectedProvider) { + request.provider = selectedProvider; + } + + const generateResponse = await EnqueueEditImage(request); + console.log("generateResponse", generateResponse); + return generateResponse; + } + + const getCompositeCanvasFile = useCallback(async (): Promise => { + if ( + !stageRef.current || + !baseImageKonvaRef.current || + !store.baseImageBitmap + ) { + return null; + } + + const editsLayer = stageRef.current.getLayers().find((l) => l.id() === DRAW_LAYER_ID); + + if (!editsLayer) { + console.error("Edits layer not found"); + return null; + } + + const rect = baseImageKonvaRef.current; + const width = rect.width(); + const height = rect.height(); + + const canvas = new OffscreenCanvas(width, height); + const ctx = canvas.getContext("2d"); + if (!ctx) return null; + + ctx.drawImage(store.baseImageBitmap, 0, 0, width, height); + + const markerLayerCanvas = editsLayer.toCanvas({ + x: stageRef.current.x(), + y: stageRef.current.y(), + width: rect.width() * stageRef.current.scaleX(), + height: rect.height() * stageRef.current.scaleY(), + pixelRatio: 1 / stageRef.current.scaleX(), + }); + const fittedMarkerCanvas = normalizeCanvas( + markerLayerCanvas, width, height, ); - if (!image) { - console.error("Failed to capture stage image"); - return; - } else { - setCanvasRenderBitmap(image); - } - }; + ctx.drawImage(fittedMarkerCanvas, 0, 0, width, height); + + const blob = await canvas.convertToBlob({ type: "image/png" }); + const uuid = crypto.randomUUID(); + return new File([blob], `${uuid}.png`, { type: "image/png" }); + }, [store.baseImageBitmap]); + + const handleGenerate = useCallback( + async ( + prompt: string, + options?: { + aspectRatio?: string; + resolution?: string; + images?: RefImage[]; + selectedProvider?: GenerationProvider; + }, + ) => { + const editedImageToken = store.baseImageInfo?.mediaToken; + + if (!editedImageToken) { + console.error("Base image is not available"); + return; + } + + // Helper to map aspect ratio string to enum + const mapAspectRatio = ( + ratio?: string, + ): EnqueueEditImageSize | undefined => { + switch (ratio) { + case "auto": + return EnqueueEditImageSize.Auto; + case "wide": + return EnqueueEditImageSize.Wide; + case "tall": + return EnqueueEditImageSize.Tall; + case "square": + return EnqueueEditImageSize.Square; + default: + return undefined; + } + }; + + // Helper to map resolution string to enum + const mapResolution = ( + res?: string, + ): EnqueueEditImageResolution | undefined => { + switch (res) { + case "1k": + return EnqueueEditImageResolution.OneK; + case "2k": + return EnqueueEditImageResolution.TwoK; + case "4k": + return EnqueueEditImageResolution.FourK; + default: + return undefined; + } + }; + + const { width, height } = store.getAspectRatioDimensions(); + const subscriberId: string = + crypto?.randomUUID?.() ?? + `inpaint-${Date.now()}-${Math.random().toString(36).slice(2)}`; + + // takes snap shot and then a global variable in the engine will invoke the inference. + const image = await captureStageEditsBitmap( + stageRef, + transformerRefs, + width, + height, + ); + + if (!image) { + console.error("Failed to capture stage edits image"); + return; + } + + try { + let result; + + if (selectedImageModel?.editingIsInpainting) { + // CASE 1 - INPAINTING (Only a few models do this!) + const arrayBuffer = await getMaskArrayBuffer(); + const request: EnqueueImageInpaintRequest = { + model: selectedImageModel, + image_media_token: editedImageToken, + mask_image_raw_bytes: arrayBuffer, + prompt: prompt, + image_count: generationCount, + frontend_caller: "image_editor", + frontend_subscriber_id: subscriberId, + }; + + if (options?.selectedProvider) { + request.provider = options.selectedProvider; + } + + result = await EnqueueImageInpaint(request); + } else if (selectedImageModel?.isNanoBananaModel()) { + // CASE 2 - NANO BANANA + const compositeFile = await getCompositeCanvasFile(); + + if (!compositeFile) { + console.error("Failed to create composite canvas"); + return; + } + + const api = new PromptsApi(); + const snapshotResult = await api.uploadSceneSnapshot({ + screenshot: compositeFile, + }); + + if (!snapshotResult.data) { + console.error("Failed to upload scene snapshot"); + return; + } + + const imgs = options?.images || []; + const request: EnqueueEditImageRequest = { + model: selectedImageModel, + scene_image_media_token: snapshotResult.data, + image_media_tokens: imgs + .map((img) => img.mediaToken) + .filter((t) => t.length > 0), + prompt: prompt, + image_count: generationCount, + frontend_caller: "image_editor", + frontend_subscriber_id: subscriberId, + aspect_ratio: mapAspectRatio(options?.aspectRatio), + image_resolution: mapResolution(options?.resolution), + }; + if (options?.selectedProvider) { + request.provider = options.selectedProvider; + } + // if (selectedImageModel?.supportsNewAspectRatio()) { + // request.common_aspect_ratio = commonAspectRatio; + // } + result = await EnqueueEditImage(request); + } else { + // CASE 3 - DEFAULT + const imgs = options?.images || []; + const request: EnqueueEditImageRequest = { + model: selectedImageModel, + image_media_tokens: [ + editedImageToken, + ...imgs + .filter((img) => img.mediaToken !== editedImageToken) + .map((img) => img.mediaToken), + ].filter((t) => t.length > 0), + disable_system_prompt: true, + prompt: prompt, + image_count: generationCount, + frontend_caller: "image_editor", + frontend_subscriber_id: subscriberId, + aspect_ratio: mapAspectRatio(options?.aspectRatio), + image_resolution: mapResolution(options?.resolution), + }; + if (options?.selectedProvider) { + request.provider = options.selectedProvider; + } + // if (selectedImageModel?.supportsNewAspectRatio()) { + // request.common_aspect_ratio = commonAspectRatio; + // } + result = await EnqueueEditImage(request); + } + if (result?.status === "success") { + setPendingGenerations((prev) => [ + ...prev, + { id: subscriberId as string, count: generationCount }, + ]); + } + } catch (error) { + setPendingGenerations((prev) => + prev.filter((p) => p.id !== subscriberId), + ); + throw error; + } + }, [generationCount, getCompositeCanvasFile, selectedImageModel, store]); const onFitPressed = async () => { // Get the stage and its container dimensions @@ -367,6 +631,7 @@ const PageDraw = () => { blurredBackgroundUrl={baseImageUrl} onImageSelect={(baseImage) => { store.clearLineNodes(); + store.setNodes([]); store.setBaseImageInfo(baseImage); }} onImageRemove={(baseImage) => { @@ -394,16 +659,6 @@ const PageDraw = () => { }`} > { - console.log("Prompt changed:", prompt); - }} - onRandomize={() => { - console.log("Randomize clicked"); - }} - onVary={() => { - console.log("Vary clicked"); - }} onAspectRatioChange={async (ratio: string) => { const ratioToType = (ratio: string): AspectRatioType => { switch (ratio) { @@ -424,8 +679,13 @@ const PageDraw = () => { await new Promise((resolve) => requestAnimationFrame(resolve)); onFitPressed(); }} - onEnqueuePressed={onEnqueuedPressed} + usePrompt2DStore={promptStoreProvider} + EncodeImageBitmapToBase64={EncodeImageBitmapToBase64} + onGenerateClick={handleGenerate} onFitPressed={onFitPressed} + isDisabled={false} + generationCount={generationCount} + onGenerationCountChange={setGenerationCount} //selectedModelInfo={selectedModelInfo} selectedImageModel={selectedImageModel} selectedProvider={selectedProvider} diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx index 3399305d60..afd597468f 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx @@ -39,6 +39,8 @@ export type MiraiProps = { const InpaintingColor = "rgba(39, 187, 245, 0.54)"; export const INPAINT_LAYER_ID = "invis-mask-layer"; +export const BG_LAYER_ID = "bg-layer"; +export const DRAW_LAYER_ID = "draw-layer"; export const PaintSurface = ({ nodes, @@ -435,7 +437,6 @@ export const PaintSurface = ({ x: isDragging.anchorX + displacement.x, y: isDragging.anchorY + displacement.y, }; - console.log(isDragging, displacement, newPos); currentStage.position(newPos); return; @@ -1503,6 +1504,7 @@ export const PaintSurface = ({ }} imageSmoothingEnabled={false} // Disable image smoothing for pixel art zIndex={-2} + id={BG_LAYER_ID} > { ctx.rect( diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/PromptEditor/PromptEditor.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/PromptEditor/PromptEditor.tsx index a90400b67c..47b9848617 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/PromptEditor/PromptEditor.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/PromptEditor/PromptEditor.tsx @@ -2,7 +2,7 @@ import React, { useState } from "react"; import { v4 as uuidv4 } from "uuid"; import ImageStyleSelector from "./ImageStyleSelector"; import { PromptEditorProps, ImageStyle } from "./types"; -import { PromptBox2D } from "@storyteller/ui-promptbox"; +import { PromptBox2D, PromptBox2DProps } from "@storyteller/ui-promptbox"; import { uploadImage } from "../../../components/reusable/UploadModalMedia/uploadImage"; import { EncodeImageBitmapToBase64 } from "../utilities/EncodeImageBitmapToBase64"; import { JobProvider, useJobContext } from "../JobContext"; @@ -10,14 +10,19 @@ import { JobProvider, useJobContext } from "../JobContext"; // Set this value on when enqueue is pressed nasty global variable. import { getCanvasRenderBitmap } from "../../../signals/canvasRenderBitmap"; -const PromptEditor: React.FC = ({ - onImageStyleChange, - onEnqueuePressed, - onAspectRatioChange, - onFitPressed, - //selectedModelInfo, +const PromptEditor: React.FC = ({ + uploadImage, selectedImageModel, selectedProvider, + EncodeImageBitmapToBase64, + onGenerateClick, + isDisabled = false, + isEnqueueing = false, + generationCount = 1, + onGenerationCountChange, + onAspectRatioChange, + onFitPressed, + usePrompt2DStore }) => { const [images, setImages] = useState([]); @@ -31,7 +36,7 @@ const PromptEditor: React.FC = ({ const updatedImages = [...images, newImage]; setImages(updatedImages); - onImageStyleChange?.(updatedImages); + // onImageStyleChange?.(updatedImages); }; return ( @@ -40,16 +45,19 @@ const PromptEditor: React.FC = ({ diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts b/frontend/apps/artcraft/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts index 4bf2a868e8..a48d22d958 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/hooks/useUpdateSnapshot.ts @@ -1,7 +1,8 @@ import { useState, useEffect } from "react"; import Konva from "konva"; +import { BG_LAYER_ID, DRAW_LAYER_ID } from "../PaintSurface"; -export const captureStageImageBitmap = async ( +export const captureStageEditsBitmap = async ( stageRef: React.RefObject, transformerRefs: React.RefObject<{ [key: string]: Konva.Transformer }>, width: number = 1024, @@ -24,6 +25,16 @@ export const captureStageImageBitmap = async ( }, ); + // Disable all the layers that we don't need + // We only need the base image layer and the edits layer + stageRef.current.getLayers().forEach((layer) => { + if (layer.id() == BG_LAYER_ID || layer.id() == DRAW_LAYER_ID) { + layer.visible(true); + } else { + layer.visible(false); + } + }); + stageRef.current.position({ x: 0, y: 0 }); stageRef.current.scale({ x: 1, y: 1 }); stageRef.current.draw(); @@ -47,6 +58,11 @@ export const captureStageImageBitmap = async ( }, ); + // Restore all layers visibility + stageRef.current.getLayers().forEach((layer) => { + layer.visible(true); + }); + stageRef.current.position(oldPos); stageRef.current.scale(oldScale); stageRef.current.draw(); diff --git a/frontend/apps/artcraft/app/src/pages/PageEdit/EditPaintSurface.tsx b/frontend/apps/artcraft/app/src/pages/PageEdit/EditPaintSurface.tsx index 1658faf76b..cdae628bb2 100644 --- a/frontend/apps/artcraft/app/src/pages/PageEdit/EditPaintSurface.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageEdit/EditPaintSurface.tsx @@ -481,11 +481,11 @@ export const EditPaintSurface = ({ setSelectionRect((prev) => prev ? { - // Ensure prev is not null - ...prev, - endX: clampedPoint.x, - endY: clampedPoint.y, - } + // Ensure prev is not null + ...prev, + endX: clampedPoint.x, + endY: clampedPoint.y, + } : null, ); } @@ -638,8 +638,8 @@ export const EditPaintSurface = ({ if (!container) return; const defaultCursor = activeTool === "edit" || - activeTool === "marker" || - activeTool === "eraser" + activeTool === "marker" || + activeTool === "eraser" ? "none" : activeTool === "select" ? "grab" @@ -693,7 +693,7 @@ export const EditPaintSurface = ({ const touch2 = touches[1]; return Math.sqrt( Math.pow(touch2.clientX - touch1.clientX, 2) + - Math.pow(touch2.clientY - touch1.clientY, 2), + Math.pow(touch2.clientY - touch1.clientY, 2), ); }; @@ -1405,31 +1405,7 @@ export const EditPaintSurface = ({ store.getAspectRatioDimensions().height, ); // leftPanelWidth, leftPanelHeight); }} - imageSmoothingEnabled={false} // Disable image smoothing for pixel art - > - - - { - ctx.rect( - 0, - 0, - store.getAspectRatioDimensions().width, - store.getAspectRatioDimensions().height, - ); // leftPanelWidth, leftPanelHeight); - }} + listening={false} > { const selectedImageModel: ImageModel | undefined = useSelectedImageModel(PAGE_ID); - const selectedProvider : GenerationProvider | undefined = + const selectedProvider: GenerationProvider | undefined = useSelectedProviderForModel(PAGE_ID, selectedImageModel?.id); - + // State for canvas dimensions const canvasWidth = useRef(1024); const canvasHeight = useRef(1024); @@ -387,7 +387,7 @@ const PageEdit = () => { if (selectedImageModel?.editingIsInpainting) { // CASE 1 - INPAINTING (Only a few models do this!) const arrayBuffer = await getMaskArrayBuffer(); - let request : EnqueueImageInpaintRequest = { + const request: EnqueueImageInpaintRequest = { model: selectedImageModel, image_media_token: editedImageToken, mask_image_raw_bytes: arrayBuffer, @@ -396,7 +396,7 @@ const PageEdit = () => { frontend_caller: "image_editor", frontend_subscriber_id: subscriberId, }; - if (!!options?.selectedProvider) { + if (options?.selectedProvider) { request.provider = options.selectedProvider; } //if (selectedImageModel?.supportsNewAspectRatio()) { @@ -419,7 +419,7 @@ const PageEdit = () => { return; } const imgs = options?.images || []; - let request: EnqueueEditImageRequest = { + const request: EnqueueEditImageRequest = { model: selectedImageModel, scene_image_media_token: snapshotResult.data, image_media_tokens: imgs @@ -432,7 +432,7 @@ const PageEdit = () => { aspect_ratio: mapAspectRatio(options?.aspectRatio), image_resolution: mapResolution(options?.resolution), }; - if (!!options?.selectedProvider) { + if (options?.selectedProvider) { request.provider = options.selectedProvider; } if (selectedImageModel?.supportsNewAspectRatio()) { @@ -442,7 +442,7 @@ const PageEdit = () => { } else { // CASE 3 - DEFAULT const imgs = options?.images || []; - let request : EnqueueEditImageRequest = { + const request: EnqueueEditImageRequest = { model: selectedImageModel, image_media_tokens: [ editedImageToken, @@ -458,7 +458,7 @@ const PageEdit = () => { aspect_ratio: mapAspectRatio(options?.aspectRatio), image_resolution: mapResolution(options?.resolution), }; - if (!!options?.selectedProvider) { + if (options?.selectedProvider) { request.provider = options.selectedProvider; } if (selectedImageModel?.supportsNewAspectRatio()) { @@ -554,9 +554,8 @@ const PageEdit = () => { {(store.activeTool === "marker" || store.activeTool === "eraser") && isNanoBananaModel && (
{ {(store.activeTool === "edit" || store.activeTool === "eraser") && supportsMaskedInpainting && (
{
)}
{ @@ -639,9 +636,8 @@ const PageEdit = () => { />