From b2a3321a0bb0de253545f6238882360cd7f58f3c Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Mon, 12 Jan 2026 14:25:41 -0800 Subject: [PATCH] Support masking and fitted resolution in 2D --- .../app/src/pages/PageDraw/PageDraw.tsx | 45 ++++- .../app/src/pages/PageDraw/PaintSurface.tsx | 155 +++++++++++++++--- .../PageDraw/components/ui/SideToolbar.tsx | 18 +- .../src/pages/PageDraw/stores/SceneState.ts | 10 ++ 4 files changed, 204 insertions(+), 24 deletions(-) diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx index e1df6fb191..617c6a6bf0 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx @@ -1,5 +1,5 @@ import { useState, useRef, useEffect } from "react"; -import { PaintSurface } from "./PaintSurface"; +import { INPAINT_LAYER_ID, PaintSurface } from "./PaintSurface"; import "./App.css"; import PromptEditor from "./PromptEditor/PromptEditor"; import SideToolbar from "./components/ui/SideToolbar"; @@ -26,6 +26,7 @@ import { HelpMenuButton } from "@storyteller/ui-help-menu"; import { GenerationProvider } from "@storyteller/api-enums"; import { HistoryStack, ImageBundle } from "../PageEdit/HistoryStack"; import { BaseImageSelector, BaseSelectorImage } from "../PageEdit/BaseImageSelector"; +import { normalizeCanvas } from "~/Helpers/CanvasHelpers"; const PAGE_ID: ModelPage = ModelPage.Canvas2D; @@ -61,6 +62,7 @@ const PageDraw = () => { const stageRef = useRef({} as Konva.Stage); const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({}); const store = useSceneStore(); + const baseImageKonvaRef = useRef({} as Konva.Image); const baseImageUrl = store.baseImageInfo?.url; const [pendingGenerations, setPendingGenerations] = useState< { id: string; count: number }[] @@ -110,6 +112,46 @@ const PageDraw = () => { ); }); + // Create a function to use the left layer ref and download the bitmap from it + const getMaskArrayBuffer = async (): Promise => { + if ( + !stageRef.current || + !baseImageKonvaRef.current + ) { + console.error("Stage or left panel ref is not available"); + throw new Error("Stage or left panel or base image ref is not available"); + } + + const layer = stageRef.current.getLayers().find((l) => l.name() === INPAINT_LAYER_ID)!; + + // Get the canvas area that's covered by the image/rectangle + const rect = baseImageKonvaRef.current; + const layerCrop = layer.toCanvas({ + x: stageRef.current.x(), + y: stageRef.current.y(), + width: rect.width() * stageRef.current.scaleX(), + height: rect.height() * stageRef.current.scaleY(), + pixelRatio: 1 / stageRef.current.scaleX(), + }); + + // Using the pixelRatio scaling may result in off-by-one rounding errors, + // So we re-fit the image to a canvas of precise size. + const fittedCanvas = normalizeCanvas( + layerCrop, + rect.width(), + rect.height(), + ); + + // Convert colored canvas to alpha mask + // NOTE: This isn't needed because the tauri backend uses the alpha channel anyway + // drawAlphaMask(fittedCanvas, rect.width(), rect.height()); + + const blob = await fittedCanvas.convertToBlob({ type: "image/png" }); + const arrayBuffer = await blob.arrayBuffer(); + + return new Uint8Array(arrayBuffer); + }; + // Listen for gallery drag and drop events useEffect(() => { const handleGallery2DDrop = (event: CustomEvent) => { @@ -523,6 +565,7 @@ const PageDraw = () => { onSelectionChange={setIsSelecting} stageRef={stageRef} transformerRefs={transformerRefs} + baseImageRef={baseImageKonvaRef} /> diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx index 3f61b941d3..3dc5badcba 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx @@ -8,6 +8,7 @@ import { Line, Image, Transformer, + Group, } from "react-konva"; import Konva from "konva"; import { LineNode, useSceneStore } from "./stores/SceneState"; @@ -18,20 +19,26 @@ import SplitPane from "./components/ui/SplitPane"; import { useRightPanelLayoutManagement } from "./hooks/useRightPanelLayoutManagement"; import { useStageCentering } from "./hooks/useCenteredStage"; import { useGlobalMouseUp } from "./hooks/useGlobalMouseUp"; +import { loadImageFromUrl } from "~/Helpers/ImageHelpers"; +import { checkerboard } from "@storyteller/common"; export type MiraiProps = { nodes: Node[]; selectedNodeIds: string[]; onCanvasSizeChange?: (width: number, height: number) => void; fillColor?: string; - activeTool?: "select" | "draw" | "eraser" | "backgroundColor" | "shape"; + activeTool?: "select" | "draw" | "eraser" | "backgroundColor" | "shape" | "inpaint"; brushColor?: string; brushSize?: number; onSelectionChange?: (isSelecting: boolean) => void; stageRef: React.RefObject; transformerRefs: React.RefObject<{ [key: string]: Konva.Transformer }>; + baseImageRef: React.RefObject; }; +const InpaintingColor = "rgba(39, 187, 245, 0.54)"; +export const INPAINT_LAYER_ID = "invis-mask-layer"; + export const PaintSurface = ({ nodes, selectedNodeIds, @@ -43,6 +50,7 @@ export const PaintSurface = ({ onSelectionChange, stageRef, transformerRefs, + baseImageRef, }: MiraiProps) => { const singlePaneMode = true; @@ -52,6 +60,11 @@ export const PaintSurface = ({ const rightContainerRef = React.useRef(null); const cursorLayerRef = React.useRef(null); const cursorShapeRef = React.useRef(null); + // TODO: Polish this by using canvas graphics instead of a bitmap + // Or at least manually do the pattern fill to avoid image interpolation + const [checkerImage, setCheckerImage] = useState( + null, + ); const containerRef = React.useRef(null); const [containerDimensions, setContainerDimensions] = useState({ @@ -204,6 +217,7 @@ export const PaintSurface = ({ activeTool !== "draw" && activeTool !== "eraser" && activeTool !== "shape" && + activeTool !== "inpaint" && nodeDraggable && !isMiddleMousePressed ); @@ -248,25 +262,45 @@ export const PaintSurface = ({ } if ( - (activeTool === "draw" || activeTool === "eraser") && + (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") && isWithinLeftPanel(stagePoint) ) { - const lineId = `line-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + const lineId = `line-${activeTool}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; const opacity = activeTool === "draw" ? store.brushOpacity : 1; + const strokeColor = + activeTool === "eraser" + ? "#FFFFFF" + : activeTool === "draw" + ? brushColor + : InpaintingColor; + + const composite = + activeTool === "eraser" + ? "destination-out" + : "source-over"; + + console.log("Starting new line with composite:", composite); + + const zindex = + activeTool === "eraser" ? + 999 : activeTool === "inpaint" ? + 998 : 1; const newLineNode: LineNode = { id: lineId, type: "line", points: [stagePoint.x, stagePoint.y], - stroke: activeTool === "draw" ? brushColor : fillColor || "#ffffff", + stroke: strokeColor, strokeWidth: brushSize / stage.scaleX(), draggable: true, opacity: opacity, locked: false, - zIndex: store.lineNodes.length, + zIndex: zindex, + globalCompositeOperation: composite, x: 0, y: 0, }; + store.selectNode(null); store.addLineNode(newLineNode, false); setCurrentLineId(lineId); @@ -359,7 +393,7 @@ export const PaintSurface = ({ const isWithinCanvas = isWithinLeftPanel(stagePoint); - if (activeTool === "draw" || activeTool === "eraser") { + if (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") { if (isWithinCanvas || isDrawing) { stage.container().style.cursor = "none"; store.setCursorPosition(pointer); @@ -410,7 +444,7 @@ export const PaintSurface = ({ if ( isDrawing && currentLineId && - (activeTool === "draw" || activeTool === "eraser") + (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") ) { const point = stage.getPointerPosition(); if (!point) return; @@ -424,6 +458,7 @@ export const PaintSurface = ({ const currentLine = store.lineNodes.find( (line) => line.id === currentLineId, ); + if (currentLine) { const updatedPoints = [ ...currentLine.points, @@ -499,10 +534,10 @@ export const PaintSurface = ({ setSelectionRect((prev) => prev ? { - ...prev, - endX: clampedPoint.x, - endY: clampedPoint.y, - } + ...prev, + endX: clampedPoint.x, + endY: clampedPoint.y, + } : null, ); } @@ -587,7 +622,7 @@ export const PaintSurface = ({ const touch2 = touches[1]; return Math.sqrt( Math.pow(touch2.clientX - touch1.clientX, 2) + - Math.pow(touch2.clientY - touch1.clientY, 2), + Math.pow(touch2.clientY - touch1.clientY, 2), ); }; @@ -719,7 +754,7 @@ export const PaintSurface = ({ const container = e.target.getStage()?.container(); if (!container) return; const defaultCursor = - activeTool === "draw" || activeTool === "eraser" + activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint" ? "none" : activeTool === "select" ? "grab" @@ -734,7 +769,7 @@ export const PaintSurface = ({ return; } - if (activeTool === "draw" || activeTool === "eraser") { + if (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") { stage.container().style.cursor = "none"; const pointer = stage.getPointerPosition(); if (pointer) { @@ -765,7 +800,7 @@ export const PaintSurface = ({ if ( store.cursorVisible && store.cursorPosition && - (activeTool === "draw" || activeTool === "eraser") + (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") ) { const stageX = stage.x(); const stageY = stage.y(); @@ -785,6 +820,10 @@ export const PaintSurface = ({ cursorNode.fill(brushColor); cursorNode.stroke("rgba(255, 255, 255, 0.7)"); cursorNode.strokeWidth(1); + } else if (activeTool === "inpaint") { + cursorNode.fill(InpaintingColor); + cursorNode.stroke("rgba(255, 255, 255, 0.7)"); + cursorNode.strokeWidth(1); } else { cursorNode.fill("rgba(255, 255, 255, 0.3)"); cursorNode.stroke("rgba(0, 0, 0, 0.7)"); @@ -811,6 +850,7 @@ export const PaintSurface = ({ if ( activeTool === "draw" || activeTool === "eraser" || + activeTool === "inpaint" || activeTool === "shape" ) { return; @@ -1050,6 +1090,10 @@ export const PaintSurface = ({ offsetX={lineNode.offsetX || 0} offsetY={lineNode.offsetY || 0} listening={listeningEnabled} + // @ts-expect-error: We don't have globalCompositeOperation in the type definition + globalCompositeOperation={ + lineNode.globalCompositeOperation || "source-over" + } /> ); } @@ -1236,6 +1280,11 @@ export const PaintSurface = ({ return null; }; + // Load the checkboard image for transparency background + useEffect(() => { + loadImageFromUrl(checkerboard).then(setCheckerImage); + }, []); + // Separate function to render transformers on their own layer const renderTransformers = () => { // Don't render individual transformers when multiple nodes are selected @@ -1398,6 +1447,18 @@ export const PaintSurface = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [selectedNodeIds]); + const inpaintingLineNodes: LineNode[] = []; + const drawLineNodes: LineNode[] = []; + + store.lineNodes.forEach((line) => { + if (line.id.startsWith("line-eraser") || line.id.startsWith("line-inpaint")) { + inpaintingLineNodes.push(line); + } + if (line.id.startsWith("line-eraser") || line.id.startsWith("line-draw")) { + drawLineNodes.push(line); + } + }); + return ( { ctx.rect( 0, 0, store.getAspectRatioDimensions().width, store.getAspectRatioDimensions().height, - ); + ); // leftPanelWidth, leftPanelHeight); }} + imageSmoothingEnabled={false} // Disable image smoothing for pixel art + zIndex={-2} > + - {[...nodes, ...store.lineNodes] - .sort((a, b) => (a.zIndex || 0) - (b.zIndex || 0)) + + + { + ctx.rect( + 0, + 0, + store.getAspectRatioDimensions().width, + store.getAspectRatioDimensions().height, + ); + }} + > + + {nodes .map((node) => renderNode(node))} + + {drawLineNodes + .map((node) => renderNode(node))} + + {selectionRect && ( )} + { + ctx.rect( + 0, + 0, + store.getAspectRatioDimensions().width, + store.getAspectRatioDimensions().height, + ); + }} + id={INPAINT_LAYER_ID} + listening={false} + > + {inpaintingLineNodes + .map((node) => renderNode(node))} + diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/components/ui/SideToolbar.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/components/ui/SideToolbar.tsx index b020fa497c..39419b07fe 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/components/ui/SideToolbar.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/components/ui/SideToolbar.tsx @@ -10,6 +10,8 @@ import { faPaintBrush, faXmark, faTrashXmark, + faEraser, + faSendBack, } from "@fortawesome/pro-solid-svg-icons"; import { faShapes, @@ -270,11 +272,19 @@ const SideToolbar: React.FC = ({ popout: BrushPopout, }, { - id: "delete", - label: "Delete", - icon: , + id: "inpaint", + label: "Mask", + icon: , onClick: () => { - onDelete(); + store.setActiveTool("inpaint"); + }, + }, + { + id: "erase", + label: "Eraser", + icon: , + onClick: () => { + store.setActiveTool("eraser"); }, }, { id: "separator-3", type: "separator" }, diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/stores/SceneState.ts b/frontend/apps/artcraft/app/src/pages/PageDraw/stores/SceneState.ts index 39c7a0a50a..907dd0041e 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/stores/SceneState.ts +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/stores/SceneState.ts @@ -57,6 +57,7 @@ export const convertFileToBase64 = (file: File): Promise => { export type ActiveTool = | "select" | "draw" + | "inpaint" | "eraser" | "backgroundColor" | "shape"; @@ -1563,6 +1564,15 @@ export const useSceneStore = create((set, get, store) => ({ }, getAspectRatioDimensions: () => { + // First we check the base image info if it exists + // Otherwise we use default landscape + const baseImageInfo = get().baseImageBitmap; + + // If base image exists, use its dimensions + if (baseImageInfo) { + return { width: baseImageInfo.width, height: baseImageInfo.height }; + } + const { aspectRatioType } = get(); switch (aspectRatioType) { case AspectRatioType.PORTRAIT: