Support masking and fitted resolution in 2D

frontend/apps/artcraft/app/src/pages/PageDraw/PageDraw.tsx

@@ -1,5 +1,5 @@
 import { useState, useRef, useEffect } from "react";
-import { PaintSurface } from "./PaintSurface";
+import { INPAINT_LAYER_ID, PaintSurface } from "./PaintSurface";
 import "./App.css";
 import PromptEditor from "./PromptEditor/PromptEditor";
 import SideToolbar from "./components/ui/SideToolbar";
@@ -26,6 +26,7 @@ import { HelpMenuButton } from "@storyteller/ui-help-menu";
 import { GenerationProvider } from "@storyteller/api-enums";
 import { HistoryStack, ImageBundle } from "../PageEdit/HistoryStack";
 import { BaseImageSelector, BaseSelectorImage } from "../PageEdit/BaseImageSelector";
+import { normalizeCanvas } from "~/Helpers/CanvasHelpers";
 
 const PAGE_ID: ModelPage = ModelPage.Canvas2D;
 
@@ -61,6 +62,7 @@ const PageDraw = () => {
   const stageRef = useRef<Konva.Stage>({} as Konva.Stage);
   const transformerRefs = useRef<{ [key: string]: Konva.Transformer }>({});
   const store = useSceneStore();
+  const baseImageKonvaRef = useRef<Konva.Image>({} as Konva.Image);
   const baseImageUrl = store.baseImageInfo?.url;
   const [pendingGenerations, setPendingGenerations] = useState<
     { id: string; count: number }[]
@@ -110,6 +112,46 @@ const PageDraw = () => {
     );
   });
 
+  // Create a function to use the left layer ref and download the bitmap from it
+  const getMaskArrayBuffer = async (): Promise<Uint8Array> => {
+    if (
+      !stageRef.current ||
+      !baseImageKonvaRef.current
+    ) {
+      console.error("Stage or left panel ref is not available");
+      throw new Error("Stage or left panel or base image ref is not available");
+    }
+
+    const layer = stageRef.current.getLayers().find((l) => l.name() === INPAINT_LAYER_ID)!;
+
+    // Get the canvas area that's covered by the image/rectangle
+    const rect = baseImageKonvaRef.current;
+    const layerCrop = layer.toCanvas({
+      x: stageRef.current.x(),
+      y: stageRef.current.y(),
+      width: rect.width() * stageRef.current.scaleX(),
+      height: rect.height() * stageRef.current.scaleY(),
+      pixelRatio: 1 / stageRef.current.scaleX(),
+    });
+
+    // Using the pixelRatio scaling may result in off-by-one rounding errors,
+    // So we re-fit the image to a canvas of precise size.
+    const fittedCanvas = normalizeCanvas(
+      layerCrop,
+      rect.width(),
+      rect.height(),
+    );
+
+    // Convert colored canvas to alpha mask
+    // NOTE: This isn't needed because the tauri backend uses the alpha channel anyway
+    // drawAlphaMask(fittedCanvas, rect.width(), rect.height());
+
+    const blob = await fittedCanvas.convertToBlob({ type: "image/png" });
+    const arrayBuffer = await blob.arrayBuffer();
+
+    return new Uint8Array(arrayBuffer);
+  };
+
   // Listen for gallery drag and drop events
   useEffect(() => {
     const handleGallery2DDrop = (event: CustomEvent) => {
@@ -523,6 +565,7 @@ const PageDraw = () => {
             onSelectionChange={setIsSelecting}
             stageRef={stageRef}
             transformerRefs={transformerRefs}
+            baseImageRef={baseImageKonvaRef}
           />
         </ContextMenuContainer>
       </div>

frontend/apps/artcraft/app/src/pages/PageDraw/PaintSurface.tsx

@@ -8,6 +8,7 @@ import {
   Line,
   Image,
   Transformer,
+  Group,
 } from "react-konva";
 import Konva from "konva";
 import { LineNode, useSceneStore } from "./stores/SceneState";
@@ -18,20 +19,26 @@ import SplitPane from "./components/ui/SplitPane";
 import { useRightPanelLayoutManagement } from "./hooks/useRightPanelLayoutManagement";
 import { useStageCentering } from "./hooks/useCenteredStage";
 import { useGlobalMouseUp } from "./hooks/useGlobalMouseUp";
+import { loadImageFromUrl } from "~/Helpers/ImageHelpers";
+import { checkerboard } from "@storyteller/common";
 
 export type MiraiProps = {
   nodes: Node[];
   selectedNodeIds: string[];
   onCanvasSizeChange?: (width: number, height: number) => void;
   fillColor?: string;
-  activeTool?: "select" | "draw" | "eraser" | "backgroundColor" | "shape";
+  activeTool?: "select" | "draw" | "eraser" | "backgroundColor" | "shape" | "inpaint";
   brushColor?: string;
   brushSize?: number;
   onSelectionChange?: (isSelecting: boolean) => void;
   stageRef: React.RefObject<Konva.Stage>;
   transformerRefs: React.RefObject<{ [key: string]: Konva.Transformer }>;
+  baseImageRef: React.RefObject<Konva.Image>;
 };
 
+const InpaintingColor = "rgba(39, 187, 245, 0.54)";
+export const INPAINT_LAYER_ID = "invis-mask-layer";
+
 export const PaintSurface = ({
   nodes,
   selectedNodeIds,
@@ -43,6 +50,7 @@ export const PaintSurface = ({
   onSelectionChange,
   stageRef,
   transformerRefs,
+  baseImageRef,
 }: MiraiProps) => {
   const singlePaneMode = true;
 
@@ -52,6 +60,11 @@ export const PaintSurface = ({
   const rightContainerRef = React.useRef<HTMLDivElement>(null);
   const cursorLayerRef = React.useRef<Konva.Layer>(null);
   const cursorShapeRef = React.useRef<Konva.Circle>(null);
+  // TODO: Polish this by using canvas graphics instead of a bitmap
+  // Or at least manually do the pattern fill to avoid image interpolation
+  const [checkerImage, setCheckerImage] = useState<HTMLImageElement | null>(
+    null,
+  );
 
   const containerRef = React.useRef<HTMLDivElement>(null);
   const [containerDimensions, setContainerDimensions] = useState({
@@ -204,6 +217,7 @@ export const PaintSurface = ({
       activeTool !== "draw" &&
       activeTool !== "eraser" &&
       activeTool !== "shape" &&
+      activeTool !== "inpaint" &&
       nodeDraggable &&
       !isMiddleMousePressed
     );
@@ -248,25 +262,45 @@ export const PaintSurface = ({
     }
 
     if (
-      (activeTool === "draw" || activeTool === "eraser") &&
+      (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") &&
       isWithinLeftPanel(stagePoint)
     ) {
-      const lineId = `line-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+      const lineId = `line-${activeTool}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
       const opacity = activeTool === "draw" ? store.brushOpacity : 1;
+      const strokeColor =
+        activeTool === "eraser"
+          ? "#FFFFFF"
+          : activeTool === "draw"
+            ? brushColor
+            : InpaintingColor;
+
+      const composite =
+        activeTool === "eraser"
+          ? "destination-out"
+          : "source-over";
+
+      console.log("Starting new line with composite:", composite);
+
+      const zindex =
+        activeTool === "eraser" ?
+          999 : activeTool === "inpaint" ?
+            998 : 1;
 
       const newLineNode: LineNode = {
         id: lineId,
         type: "line",
         points: [stagePoint.x, stagePoint.y],
-        stroke: activeTool === "draw" ? brushColor : fillColor || "#ffffff",
+        stroke: strokeColor,
         strokeWidth: brushSize / stage.scaleX(),
         draggable: true,
         opacity: opacity,
         locked: false,
-        zIndex: store.lineNodes.length,
+        zIndex: zindex,
+        globalCompositeOperation: composite,
         x: 0,
         y: 0,
       };
+
       store.selectNode(null);
       store.addLineNode(newLineNode, false);
       setCurrentLineId(lineId);
@@ -359,7 +393,7 @@ export const PaintSurface = ({
 
       const isWithinCanvas = isWithinLeftPanel(stagePoint);
 
-      if (activeTool === "draw" || activeTool === "eraser") {
+      if (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") {
         if (isWithinCanvas || isDrawing) {
           stage.container().style.cursor = "none";
           store.setCursorPosition(pointer);
@@ -410,7 +444,7 @@ export const PaintSurface = ({
     if (
       isDrawing &&
       currentLineId &&
-      (activeTool === "draw" || activeTool === "eraser")
+      (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint")
     ) {
       const point = stage.getPointerPosition();
       if (!point) return;
@@ -424,6 +458,7 @@ export const PaintSurface = ({
       const currentLine = store.lineNodes.find(
         (line) => line.id === currentLineId,
       );
+
       if (currentLine) {
         const updatedPoints = [
           ...currentLine.points,
@@ -499,10 +534,10 @@ export const PaintSurface = ({
       setSelectionRect((prev) =>
         prev
           ? {
-              ...prev,
-              endX: clampedPoint.x,
-              endY: clampedPoint.y,
-            }
+            ...prev,
+            endX: clampedPoint.x,
+            endY: clampedPoint.y,
+          }
           : null,
       );
     }
@@ -587,7 +622,7 @@ export const PaintSurface = ({
     const touch2 = touches[1];
     return Math.sqrt(
       Math.pow(touch2.clientX - touch1.clientX, 2) +
-        Math.pow(touch2.clientY - touch1.clientY, 2),
+      Math.pow(touch2.clientY - touch1.clientY, 2),
     );
   };
 
@@ -719,7 +754,7 @@ export const PaintSurface = ({
     const container = e.target.getStage()?.container();
     if (!container) return;
     const defaultCursor =
-      activeTool === "draw" || activeTool === "eraser"
+      activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint"
         ? "none"
         : activeTool === "select"
           ? "grab"
@@ -734,7 +769,7 @@ export const PaintSurface = ({
       return;
     }
 
-    if (activeTool === "draw" || activeTool === "eraser") {
+    if (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint") {
       stage.container().style.cursor = "none";
       const pointer = stage.getPointerPosition();
       if (pointer) {
@@ -765,7 +800,7 @@ export const PaintSurface = ({
     if (
       store.cursorVisible &&
       store.cursorPosition &&
-      (activeTool === "draw" || activeTool === "eraser")
+      (activeTool === "draw" || activeTool === "eraser" || activeTool === "inpaint")
     ) {
       const stageX = stage.x();
       const stageY = stage.y();
@@ -785,6 +820,10 @@ export const PaintSurface = ({
         cursorNode.fill(brushColor);
         cursorNode.stroke("rgba(255, 255, 255, 0.7)");
         cursorNode.strokeWidth(1);
+      } else if (activeTool === "inpaint") {
+        cursorNode.fill(InpaintingColor);
+        cursorNode.stroke("rgba(255, 255, 255, 0.7)");
+        cursorNode.strokeWidth(1);
       } else {
         cursorNode.fill("rgba(255, 255, 255, 0.3)");
         cursorNode.stroke("rgba(0, 0, 0, 0.7)");
@@ -811,6 +850,7 @@ export const PaintSurface = ({
       if (
         activeTool === "draw" ||
         activeTool === "eraser" ||
+        activeTool === "inpaint" ||
         activeTool === "shape"
       ) {
         return;
@@ -1050,6 +1090,10 @@ export const PaintSurface = ({
           offsetX={lineNode.offsetX || 0}
           offsetY={lineNode.offsetY || 0}
           listening={listeningEnabled}
+          // @ts-expect-error: We don't have globalCompositeOperation in the type definition
+          globalCompositeOperation={
+            lineNode.globalCompositeOperation || "source-over"
+          }
         />
       );
     }
@@ -1236,6 +1280,11 @@ export const PaintSurface = ({
     return null;
   };
 
+  // Load the checkboard image for transparency background
+  useEffect(() => {
+    loadImageFromUrl(checkerboard).then(setCheckerImage);
+  }, []);
+
   // Separate function to render transformers on their own layer
   const renderTransformers = () => {
     // Don't render individual transformers when multiple nodes are selected
@@ -1398,6 +1447,18 @@ export const PaintSurface = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [selectedNodeIds]);
 
+  const inpaintingLineNodes: LineNode[] = [];
+  const drawLineNodes: LineNode[] = [];
+
+  store.lineNodes.forEach((line) => {
+    if (line.id.startsWith("line-eraser") || line.id.startsWith("line-inpaint")) {
+      inpaintingLineNodes.push(line);
+    }
+    if (line.id.startsWith("line-eraser") || line.id.startsWith("line-draw")) {
+      drawLineNodes.push(line);
+    }
+  });
+
   return (
     <SplitPane
       singlePaneMode={singlePaneMode}
@@ -1431,16 +1492,30 @@ export const PaintSurface = ({
             onTouchEnd={handleTouchEnd}
           >
             <Layer
-              ref={leftPanelRef}
               clipFunc={(ctx) => {
                 ctx.rect(
                   0,
                   0,
                   store.getAspectRatioDimensions().width,
                   store.getAspectRatioDimensions().height,
-                );
+                ); // leftPanelWidth, leftPanelHeight);
               }}
+              imageSmoothingEnabled={false} // Disable image smoothing for pixel art
+              zIndex={-2}
             >
+              <Rect
+                x={0}
+                y={0}
+                fillPatternImage={checkerImage || undefined}
+                fillPatternRepeat="repeat"
+                fillPatternScaleX={window.devicePixelRatio}
+                fillPatternScaleY={window.devicePixelRatio}
+                width={store.getAspectRatioDimensions().width}
+                height={store.getAspectRatioDimensions().height}
+                listening={false}
+                imageSmoothingEnabled={false} // Disable image smoothing for pixel art
+                zIndex={-2}
+              />
               <Rect
                 x={0}
                 y={0}
@@ -1451,10 +1526,37 @@ export const PaintSurface = ({
                 zIndex={-1}
               />
 
-              {[...nodes, ...store.lineNodes]
-                .sort((a, b) => (a.zIndex || 0) - (b.zIndex || 0))
+              <Image
+                ref={baseImageRef}
+                x={0}
+                y={0}
+                image={store.baseImageBitmap || undefined}
+                width={store.getAspectRatioDimensions().width}
+                height={store.getAspectRatioDimensions().height}
+                listening={false}
+              />
+            </Layer>
+            <Layer
+              ref={leftPanelRef}
+              clipFunc={(ctx) => {
+                ctx.rect(
+                  0,
+                  0,
+                  store.getAspectRatioDimensions().width,
+                  store.getAspectRatioDimensions().height,
+                );
+              }}
+            >
+
+              {nodes
                 .map((node) => renderNode(node))}
 
+              <Group
+                cach>
+                {drawLineNodes
+                  .map((node) => renderNode(node))}
+              </Group>
+
               {selectionRect && (
                 <Rect
                   ref={selectionRectRef}
@@ -1483,6 +1585,21 @@ export const PaintSurface = ({
                 />
               )}
             </Layer>
+            <Layer
+              clipFunc={(ctx) => {
+                ctx.rect(
+                  0,
+                  0,
+                  store.getAspectRatioDimensions().width,
+                  store.getAspectRatioDimensions().height,
+                );
+              }}
+              id={INPAINT_LAYER_ID}
+              listening={false}
+            >
+              {inpaintingLineNodes
+                .map((node) => renderNode(node))}
+            </Layer>
             <Layer ref={cursorLayerRef} listening={false} draggable={false}>
               <Circle ref={cursorShapeRef} visible={false} />
             </Layer>

frontend/apps/artcraft/app/src/pages/PageDraw/components/ui/SideToolbar.tsx

@@ -10,6 +10,8 @@ import {
   faPaintBrush,
   faXmark,
   faTrashXmark,
+  faEraser,
+  faSendBack,
 } from "@fortawesome/pro-solid-svg-icons";
 import {
   faShapes,
@@ -270,11 +272,19 @@ const SideToolbar: React.FC<SideToolbarProps> = ({
       popout: BrushPopout,
     },
     {
-      id: "delete",
-      label: "Delete",
-      icon: <FontAwesomeIcon icon={faTrash} className="h-5 w-5" />,
+      id: "inpaint",
+      label: "Mask",
+      icon: <FontAwesomeIcon icon={faSendBack} className="h-5 w-5" />,
       onClick: () => {
-        onDelete();
+        store.setActiveTool("inpaint");
+      },
+    },
+    {
+      id: "erase",
+      label: "Eraser",
+      icon: <FontAwesomeIcon icon={faEraser} className="h-5 w-5" />,
+      onClick: () => {
+        store.setActiveTool("eraser");
       },
     },
     { id: "separator-3", type: "separator" },

frontend/apps/artcraft/app/src/pages/PageDraw/stores/SceneState.ts

@@ -57,6 +57,7 @@ export const convertFileToBase64 = (file: File): Promise<string> => {
 export type ActiveTool =
   | "select"
   | "draw"
+  | "inpaint"
   | "eraser"
   | "backgroundColor"
   | "shape";
@@ -1563,6 +1564,15 @@ export const useSceneStore = create<SceneState>((set, get, store) => ({
   },
 
   getAspectRatioDimensions: () => {
+    // First we check the base image info if it exists
+    // Otherwise we use default landscape
+    const baseImageInfo = get().baseImageBitmap;
+
+    // If base image exists, use its dimensions
+    if (baseImageInfo) {
+      return { width: baseImageInfo.width, height: baseImageInfo.height };
+    }
+
     const { aspectRatioType } = get();
     switch (aspectRatioType) {
       case AspectRatioType.PORTRAIT: