Merge branch 'main' into feature/shapes-button

frontend/realtime/src/KonvaApp/Engine.ts

@@ -2,6 +2,7 @@ import Konva from "konva";
 
 
 import { uiAccess, uiEvents } from "~/signals";
+import { ShapeNode } from "./Nodes";
 
 import { UndoStackManager } from "./UndoRedo";
 import { CommandManager, MatteBox, SceneManager } from "./EngineUtitlities";
@@ -26,7 +27,7 @@ import { NavigateFunction } from "react-router-dom";
 import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider";
 
 import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler";
-import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine";
+import { RealTimeDrawEngine, ShapeType } from "./RenderingPrimitives/RealTimeDrawEngine";
 
 // for testing loading files from system
 // import { FileUtilities } from "./FileUtilities/FileUtilities";
@@ -97,6 +98,7 @@ export class Engine {
     this.stage.add(this.nodeIsolationLayer);
     this.stage.add(this.uiLayer);
 
+
     // Konva Transformer
     this.nodeTransformer = new NodeTransformer();
     this.uiLayer.add(this.nodeTransformer.getKonvaNode());
@@ -177,6 +179,11 @@ export class Engine {
     // hence, lastly, setup these events
     this.setupEventSystem();
     this.setAppMode(AppModes.SELECT);
+
+  
+    this.addShape(ShapeType.Circle,100);
+    this.addShape(ShapeType.Square,100);
+    this.addShape(ShapeType.Triangle,100);
   }
 
 
@@ -530,6 +537,22 @@ export class Engine {
     this.commandManager.createNode(textNode);
   }
 
+  public addShape(type: ShapeType, size: number, color?: string) {
+    const shapeNode = new ShapeNode({
+      canvasPosition: this.realTimeDrawEngine.captureCanvas.position(),
+      canvasSize: this.realTimeDrawEngine.captureCanvas.size(),
+      shapeType: type,
+      size: size,
+      color: color,
+      mediaLayerRef: this.mediaLayer,
+      selectionManagerRef: this.selectionManager
+    });
+    shapeNode.kNode.zIndex(1); // Replace desiredZIndex with a number
+   
+    this.commandManager.createNode(shapeNode);
+    //this.realTimeDrawEngine.addNodes(shapeNode);
+  }
+
   public addImage(imageFile: File) {
     const imageNode = new ImageNode({
       mediaLayerRef: this.mediaLayer,
@@ -538,9 +561,9 @@ export class Engine {
       imageFile: imageFile,
       selectionManagerRef: this.selectionManager,
     });
-
+    imageNode.kNode.zIndex(1);
     this.commandManager.createNode(imageNode);
-    this.realTimeDrawEngine.addNodes(imageNode);
+    //this.realTimeDrawEngine.addNodes(imageNode);
   }
 
   public addVideo(

frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts

@@ -1,5 +1,5 @@
 import Konva from "konva";
-import { ImageNode, TextNode, VideoNode } from "../Nodes";
+import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes";
 import { MediaNode, Position, RGBColor, Transformation } from "../types";
 import {
   AddChromaKey,
@@ -48,7 +48,7 @@ export class CommandManager {
     this.undoStackManagerRef = engineRefs.undoStackManagerRef;
   }
 
-  createNode(node: VideoNode | ImageNode | TextNode) {
+  createNode(node: VideoNode | ImageNode | TextNode | ShapeNode) {
     const command = new CreateCommand({
       nodes: new Set<MediaNode>([node]),
       mediaLayerRef: this.mediaLayerRef,

frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts

@@ -29,7 +29,7 @@ export abstract class BaseNode {
   }: {
     selectionManagerRef: SelectionManager;
     mediaLayerRef: Konva.Layer;
-    kNode: Konva.Image | Konva.Group;
+    kNode: Konva.Image | Konva.Group | Konva.Shape;
   }) {
     // console.log("Node constructed");
     // this.uuid = uuidv4();
@@ -67,8 +67,11 @@ export abstract class BaseNode {
     }
     if (this.kNode instanceof Konva.Group) {
       const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect;
-      wrapperRect.stroke(primaryOrange);
-      wrapperRect.strokeWidth(highlightStrokeWidth);
+      console.log("Wrapper rect:", wrapperRect);
+      if (wrapperRect !=null) {
+        wrapperRect.stroke(primaryOrange);
+        wrapperRect.strokeWidth(highlightStrokeWidth);
+      }
       return;
     }
     if (import.meta.env.DEV) {
@@ -86,7 +89,11 @@ export abstract class BaseNode {
     }
     if (this.kNode instanceof Konva.Group) {
       const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect;
-      wrapperRect.strokeWidth(0);
+      console.log("Wrapper rect:", wrapperRect); 
+      if (wrapperRect !=null) {
+        // Issue investigate
+        wrapperRect.strokeWidth(0);
+      }
       return;
     }
     if (import.meta.env.DEV) {

frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts

@@ -83,8 +83,10 @@ export class ImageNode extends NetworkedNode {
   }) {
     const imageComponent = new Image();
     imageComponent.crossOrigin = "anonymous";
+    // this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
+
     imageComponent.onload = () => {
-      this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED });
+      //this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED });
       this.imageSize = {
         width: imageComponent.width,
         height: imageComponent.height,
@@ -105,33 +107,33 @@ export class ImageNode extends NetworkedNode {
       this.kNode.fill(transparent);
       this.listenToBaseKNode();
       this.mediaLayerRef.draw();
-      this.uploadImage(imageFile);
+      //this.uploadImage(imageFile);
     };
     imageComponent.onerror = () => {
-      this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE });
+      //this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE });
     };
     imageComponent.src = URL.createObjectURL(imageFile);
   }
   private async loadImageFromUrl(mediaFileUrl: string) {
-    this.setProgress({ progress: 75, status: UploadStatus.LOADING });
+    //this.setProgress({ progress: 75, status: UploadStatus.LOADING });
 
     const newImage = new Image();
     newImage.crossOrigin = "anonymous";
     newImage.onerror = () => {
-      this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD });
+      //this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD });
     };
     newImage.onload = () => {
       // console.log("network image", newImage);
       this.kNode.image(newImage);
       this.kNode.draw();
-      this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
+      //this.setProgress({ progress: 100, status: UploadStatus.SUCCESS });
     };
     newImage.src = mediaFileUrl;
     this.listenToBaseKNode();
   }
 
   private async uploadImage(imageFile: File) {
-    this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });
+    //this.setProgress({ progress: 10, status: UploadStatus.UPLOADING });
 
     const mediaUploadApi = new MediaUploadApi();
     const uploadResponse = await mediaUploadApi.UploadImage({
@@ -149,7 +151,7 @@ export class ImageNode extends NetworkedNode {
   }
 
   private async retreiveImage(mediaFileToken: string) {
-    this.setProgress({ progress: 50, status: UploadStatus.RETREIVING });
+    //this.setProgress({ progress: 50, status: UploadStatus.RETREIVING });
     const mediaFileApi = new MediaFilesApi();
     const mediaFileResponse = await mediaFileApi.GetMediaFileByToken({
       mediaFileToken: mediaFileToken,

frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts

@@ -0,0 +1,194 @@
+import Konva from "konva";
+import { SelectionManager } from "../NodesManagers";
+import { Position, Size, NodeData, TransformationData } from "../types";
+import { BaseNode } from "./BaseNode";
+import { NodeType } from "./constants";
+import { NodeUtilities } from "./NodeUtilities";
+
+export enum ShapeType {
+  CIRCLE = "circle",
+  SQUARE = "square",
+  TRIANGLE = "triangle",
+}
+
+interface ShapeNodeConstructor {
+  canvasPosition: Position;
+  canvasSize: Size;
+  shapeType: ShapeType;
+  size: number;
+  color?: string; // Hex color string
+  transform?: TransformationData;
+  mediaLayerRef: Konva.Layer;
+  selectionManagerRef: SelectionManager;
+}
+
+export class ShapeNode extends BaseNode {
+  public kNode: Konva.Group;
+
+  constructor({
+    canvasPosition,
+    canvasSize,
+    shapeType,
+    size,
+    color = "#ff0000", // Default red if no color provided
+    transform: existingTransform,
+    mediaLayerRef,
+    selectionManagerRef,
+  }: ShapeNodeConstructor) {
+    // Create the actual shape inside the group
+    const transform = NodeUtilities.getInitialTransform({
+      existingTransform,
+      canvasPosition,
+      canvasSize,
+    });
+    let shape = null;
+    switch (shapeType) {
+      case ShapeType.CIRCLE:
+        const circleShape = new Konva.Circle({
+          radius: 50,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert circle to image
+        const dataURL = circleShape.toDataURL();
+        const imageObj = new Image();
+        imageObj.src = dataURL;
+
+        shape = new Konva.Image({
+          image: imageObj,
+          x: circleShape.x(),
+          y: circleShape.y(),
+          width: circleShape.width(),
+          height: circleShape.height(),
+          ...transform,
+          fill: 'transparent',
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        imageObj.onload = () => {
+          this.mediaLayerRef.draw();
+        };
+
+        // Clean up the temporary circle
+        circleShape.destroy();
+        break;
+
+      case ShapeType.SQUARE:
+        const squareShape = new Konva.Rect({
+          width: 100,
+          height: 100,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert square to image
+        const squareDataURL = squareShape.toDataURL();
+        const squareImageObj = new Image();
+        squareImageObj.src = squareDataURL;
+
+        shape = new Konva.Image({
+          image: squareImageObj,
+          x: squareShape.x(),
+          y: squareShape.y(),
+          width: squareShape.width(),
+          height: squareShape.height(),
+          ...transform,
+          fill: 'transparent',
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        squareImageObj.onload = () => {
+          this.mediaLayerRef.draw();
+        };
+
+        // Clean up the temporary square
+        squareShape.destroy();
+        break;
+
+      case ShapeType.TRIANGLE:
+        const triangleShape = new Konva.RegularPolygon({
+          sides: 3,
+          radius: 50,
+          ...transform,
+          fill: color,
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        // Convert triangle to image
+        const triangleDataURL = triangleShape.toDataURL();
+        const triangleImageObj = new Image();
+        triangleImageObj.src = triangleDataURL;
+
+        shape = new Konva.Image({
+          image: triangleImageObj,
+          x: triangleShape.x(),
+          y: triangleShape.y(),
+          width: triangleShape.width(),
+          height: triangleShape.height(),
+          ...transform,
+          fill: 'transparent',
+          strokeScaleEnabled: false,
+          draggable: true,
+        });
+
+        triangleImageObj.onload = () => {
+          this.mediaLayerRef.draw();
+        };
+
+        // Clean up the temporary triangle
+        triangleShape.destroy();
+        break;
+
+      default:
+        throw new Error("Invalid shape type");
+    }
+
+    super({
+      selectionManagerRef: selectionManagerRef,
+      mediaLayerRef: mediaLayerRef,
+      kNode: shape,
+    });
+
+    this.kNode = shape;
+    this.shapeType = shapeType;
+
+    // Add shape to group
+    const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({
+      canvasOffset: canvasPosition,
+      componentSize: canvasSize,
+      maxSize: canvasSize,
+    });
+    this.kNode.setPosition(centerPosition);
+    this.mediaLayerRef.add(this.kNode);
+    this.listenToBaseKNode();
+    this.mediaLayerRef.draw();
+  }
+
+  public getNodeData(canvasPosition: Position) {
+    const data: NodeData = {
+      type: NodeType.SHAPE,
+      transform: {
+        position: {
+          x: this.kNode.position().x - canvasPosition.x,
+          y: this.kNode.position().y - canvasPosition.y,
+        },
+        size: this.kNode.size(),
+        rotation: this.kNode.rotation(),
+        scale: {
+          x: this.kNode.scaleX(),
+          y: this.kNode.scaleY(),
+        },
+        zIndex: this.kNode.getZIndex(),
+      },
+    };
+    return data;
+  }
+}

frontend/realtime/src/KonvaApp/Nodes/constants.ts

@@ -4,6 +4,7 @@ export enum NodeType {
   IMAGE = "image",
   TEXT = "text",
   VIDEO = "video",
+  SHAPE = "shape"
 }
 export const transparent = Colors.transparent;
 export const primaryOrange = Colors.primaryOrange;

frontend/realtime/src/KonvaApp/Nodes/index.ts

@@ -3,3 +3,4 @@ export { BaseNode } from "./BaseNode";
 export { ImageNode } from "./ImageNode";
 export { VideoNode } from "./VideoNode";
 export { TextNode } from "./TextNode";
+export { ShapeNode } from "./ShapeNode";

frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts

@@ -3,22 +3,24 @@ import { Container } from "konva/lib/Container";
 import { Shape } from "konva/lib/Shape";
 import { Group } from "konva/lib/Group";
 
-
 import { invoke } from "@tauri-apps/api/core";
 
-
 import { FileUtilities } from "../FileUtilities/FileUtilities";
-import { ImageNode, VideoNode, TextNode } from "../Nodes";
+import { ImageNode, VideoNode, TextNode, ShapeNode } from "../Nodes";
 import { MediaNode } from "../types";
 
 import { RenderTask } from "./RenderTask";
 import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas";
 
 // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
-
+export enum ShapeType {
+  Square = "square",
+  Triangle = "triangle", 
+  Circle = "circle"
+}
 export class RealTimeDrawEngine {
   private videoNodes: VideoNode[];
-  private imageNodes: (ImageNode | TextNode)[];
+  private imageNodes: (ImageNode | TextNode | ShapeNode)[];
 
   private offScreenCanvas: OffscreenCanvas;
   private outputBitmap: ImageBitmap | undefined;
@@ -26,7 +28,7 @@ export class RealTimeDrawEngine {
   // private frames: ImageBitmap[];
 
   // capturing composite within window
-  private bgLayerRef: Konva.Layer;
+
   private mediaLayerRef: Konva.Layer;
 
   private height: number;
@@ -51,13 +53,11 @@ export class RealTimeDrawEngine {
   constructor({
     width,
     height,
-    bgLayerRef,
     mediaLayerRef,
     offScreenCanvas,
   }: {
     width: number;
     height: number;
-    bgLayerRef: Konva.Layer;
     mediaLayerRef: Konva.Layer;
     offScreenCanvas: OffscreenCanvas;
   }) {
@@ -80,7 +80,6 @@ export class RealTimeDrawEngine {
     this.offScreenCanvas.width = this.width;
     this.offScreenCanvas.height = this.height;
 
-    this.bgLayerRef = bgLayerRef;
 
     // this is the whole canvas
     this.mediaLayerRef = mediaLayerRef;
@@ -90,7 +89,7 @@ export class RealTimeDrawEngine {
     this.port = undefined;
 
     this.fps = 24;
-    
+
     this.currentPrompt = "";
     this.currentStrength = 100;
     // This is captures a subset of the medialayer ref
@@ -106,20 +105,6 @@ export class RealTimeDrawEngine {
       draggable: false,
     });
 
-    // Add preview canvas with same dimensions but different style
-    // this.previewCanvas = new Konva.Rect({
-    //   name: "PreviewCanvas",
-    //   x: this.positionX,
-    //   y: this.positionY,
-    //   width: this.width,
-    //   height: this.height,
-    //   fill: "white",
-    //   stroke: "blue",
-    //   strokeWidth: 1,
-    //   dash: [5, 5],
-    //   draggable: false,
-    // });
-
     this.previewCanvas = new Konva.Image({
       name: "PreviewCanvas",
       x: this.positionX,
@@ -127,19 +112,17 @@ export class RealTimeDrawEngine {
       width: this.width,
       height: this.height,
       image: undefined,
-      stroke: "blue",
+      stroke: "black",
       strokeWidth: 1,
-      dash: [5, 5],
       draggable: false,
+      fill: "white",
     });
 
-    this.bgLayerRef.add(this.captureCanvas);
-    this.bgLayerRef.add(this.previewCanvas);
+    this.mediaLayerRef.add(this.captureCanvas);
+    this.mediaLayerRef.add(this.previewCanvas);
     // send back
     this.captureCanvas.setZIndex(0);
-    this.previewCanvas.setZIndex(0);
-
-    //this.debug();
+    this.previewCanvas.setZIndex(1);
   }
 
   async updateCaptureCanvas(
@@ -284,12 +267,13 @@ export class RealTimeDrawEngine {
     this.isProcessing = true;
     await this.render();
   };
+  
   public addNodes(node: MediaNode) {
-    if (node instanceof VideoNode) {
-      this.videoNodes.push(node);
-      node.kNode.on("dragend", this.handleNodeDragEnd);
-    } else if (node instanceof ImageNode || node instanceof TextNode) {
+   
+    if (node instanceof ImageNode || node instanceof TextNode || node instanceof ShapeNode) {
+      console.debug("Adding node:", node);
       this.imageNodes.push(node);
+      console.log(this.imageNodes)
       node.kNode.on("dragend", this.handleNodeDragEnd);
     }
   }
@@ -325,6 +309,7 @@ export class RealTimeDrawEngine {
     try {
       const box = config.layerOfInterest.getClientRect();
       const stage = config.layerOfInterest.getStage();
+      
 
       const x = config.x !== undefined ? config.x : Math.floor(box.x);
       const y = config.y !== undefined ? config.y : Math.floor(box.y);
@@ -384,8 +369,6 @@ export class RealTimeDrawEngine {
     }
   }
 
-
-
   private async imageBitmapToBase64(imageBitmap: ImageBitmap): Promise<string> {
     // Create a temporary canvas
     const canvas = document.createElement("canvas");
@@ -462,9 +445,11 @@ export class RealTimeDrawEngine {
   }
 
   public async render() {
-    // only pick nodes that intersect with the canvas on screen bounds to freeze.
+    // only pick nodes that intersect wi th the canvas on screen bounds to freeze.
     this.mediaLayerRef.draw();
-
+    // Output all nodes in mediaLayerRef
+    const nodes = this.mediaLayerRef.getChildren();
+    console.log("All nodes in mediaLayer:", nodes);
     console.log(
       `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`,
     );
@@ -482,20 +467,20 @@ export class RealTimeDrawEngine {
     })) as ImageBitmap;
 
     // Test code
-    if (false) {
+    if (true) {
       this.outputBitmap = bitmap;
       this.previewCanvas.image(bitmap);
       this.isProcessing = false;
-      return
-    } 
+      return;
+    }
 
     try {
       const base64Bitmap = await this.imageBitmapToBase64(bitmap);
 
       const base64BitmapResponse = await invoke("infer_image", {
         image: base64Bitmap,
-        prompt:this.currentPrompt,
-        strength:this.currentStrength,
+        prompt: this.currentPrompt,
+        strength: this.currentStrength,
       });
 
       console.log(base64BitmapResponse);
@@ -510,6 +495,5 @@ export class RealTimeDrawEngine {
     } finally {
       this.isProcessing = false;
     }
-
   }
 }

frontend/realtime/src/KonvaApp/types/NodeData.ts

@@ -1,4 +1,4 @@
-import { NodeType, BaseNode, ImageNode, TextNode, VideoNode } from "../Nodes";
+import { NodeType, BaseNode, ImageNode, TextNode, VideoNode, ShapeNode } from "../Nodes";
 
 import { TransformationData } from "./Transformation";
 
@@ -6,7 +6,7 @@ import { TextNodeData } from "./Text";
 
 export type MediaNode =
   // | NetworkedNode
-  BaseNode | VideoNode | ImageNode | TextNode;
+  BaseNode | VideoNode | ImageNode | TextNode | ShapeNode;
 
 export type ImageNodeData = {
   mediaFileUrl: string;

frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx

@@ -42,12 +42,6 @@ export const DialogAddImage = ({
     closeCallback();
   }
 
-  function handleEnter() {
-    if (currFile) {
-      onAddImage(currFile);
-    }
-    handleClose();
-  }
   return (
     <Dialog open={isOpen} onClose={closeCallback} className="relative z-50">
       <div className={dialogBackgroundStyles}>
@@ -66,6 +60,10 @@ export const DialogAddImage = ({
               file={currFile}
               setFile={(file: File | null) => {
                 setAssetFile(file);
+                if (file) {
+                  onAddImage(file);
+                  handleClose();
+                }
               }}
             />
             {currFile && (
@@ -84,9 +82,6 @@ export const DialogAddImage = ({
             <Button onClick={handleClose} variant="secondary">
               Cancel
             </Button>
-            <Button onClick={handleEnter} disabled={currFile === null}>
-              Enter
-            </Button>
           </div>
         </DialogPanel>
       </div>

frontend/realtime/src/signals/uiEvents/addMedia.ts

@@ -6,6 +6,7 @@ const stagedImage = signal<File | null>(null);
 const addImageToEngine = (image: File) => {
   stagedImage.value = image;
 };
+
 const onGetStagedImage = (callback: (file: File) => void) => {
   effect(() => {
     if (stagedImage.value) {