diff --git a/frontend/realtime/src/KonvaApp/Engine.ts b/frontend/realtime/src/KonvaApp/Engine.ts index 2114b570a3..7be2a9ce05 100644 --- a/frontend/realtime/src/KonvaApp/Engine.ts +++ b/frontend/realtime/src/KonvaApp/Engine.ts @@ -2,6 +2,7 @@ import Konva from "konva"; import { uiAccess, uiEvents } from "~/signals"; +import { ShapeNode } from "./Nodes"; import { UndoStackManager } from "./UndoRedo"; import { CommandManager, MatteBox, SceneManager } from "./EngineUtitlities"; @@ -26,7 +27,7 @@ import { NavigateFunction } from "react-router-dom"; import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider"; import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler"; -import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine"; +import { RealTimeDrawEngine, ShapeType } from "./RenderingPrimitives/RealTimeDrawEngine"; // for testing loading files from system // import { FileUtilities } from "./FileUtilities/FileUtilities"; @@ -97,6 +98,7 @@ export class Engine { this.stage.add(this.nodeIsolationLayer); this.stage.add(this.uiLayer); + // Konva Transformer this.nodeTransformer = new NodeTransformer(); this.uiLayer.add(this.nodeTransformer.getKonvaNode()); @@ -177,6 +179,11 @@ export class Engine { // hence, lastly, setup these events this.setupEventSystem(); this.setAppMode(AppModes.SELECT); + + + this.addShape(ShapeType.Circle,100); + this.addShape(ShapeType.Square,100); + this.addShape(ShapeType.Triangle,100); } @@ -530,6 +537,22 @@ export class Engine { this.commandManager.createNode(textNode); } + public addShape(type: ShapeType, size: number, color?: string) { + const shapeNode = new ShapeNode({ + canvasPosition: this.realTimeDrawEngine.captureCanvas.position(), + canvasSize: this.realTimeDrawEngine.captureCanvas.size(), + shapeType: type, + size: size, + color: color, + mediaLayerRef: this.mediaLayer, + selectionManagerRef: this.selectionManager + }); + shapeNode.kNode.zIndex(1); // Replace desiredZIndex with a number + + this.commandManager.createNode(shapeNode); + //this.realTimeDrawEngine.addNodes(shapeNode); + } + public addImage(imageFile: File) { const imageNode = new ImageNode({ mediaLayerRef: this.mediaLayer, @@ -538,9 +561,9 @@ export class Engine { imageFile: imageFile, selectionManagerRef: this.selectionManager, }); - + imageNode.kNode.zIndex(1); this.commandManager.createNode(imageNode); - this.realTimeDrawEngine.addNodes(imageNode); + //this.realTimeDrawEngine.addNodes(imageNode); } public addVideo( diff --git a/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts b/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts index cc4862f536..bb008055f2 100644 --- a/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts +++ b/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts @@ -1,5 +1,5 @@ import Konva from "konva"; -import { ImageNode, TextNode, VideoNode } from "../Nodes"; +import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes"; import { MediaNode, Position, RGBColor, Transformation } from "../types"; import { AddChromaKey, @@ -48,7 +48,7 @@ export class CommandManager { this.undoStackManagerRef = engineRefs.undoStackManagerRef; } - createNode(node: VideoNode | ImageNode | TextNode) { + createNode(node: VideoNode | ImageNode | TextNode | ShapeNode) { const command = new CreateCommand({ nodes: new Set([node]), mediaLayerRef: this.mediaLayerRef, diff --git a/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts b/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts index f9e60a9983..1ca2cd8c83 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts @@ -29,7 +29,7 @@ export abstract class BaseNode { }: { selectionManagerRef: SelectionManager; mediaLayerRef: Konva.Layer; - kNode: Konva.Image | Konva.Group; + kNode: Konva.Image | Konva.Group | Konva.Shape; }) { // console.log("Node constructed"); // this.uuid = uuidv4(); @@ -67,8 +67,11 @@ export abstract class BaseNode { } if (this.kNode instanceof Konva.Group) { const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect; - wrapperRect.stroke(primaryOrange); - wrapperRect.strokeWidth(highlightStrokeWidth); + console.log("Wrapper rect:", wrapperRect); + if (wrapperRect !=null) { + wrapperRect.stroke(primaryOrange); + wrapperRect.strokeWidth(highlightStrokeWidth); + } return; } if (import.meta.env.DEV) { @@ -86,7 +89,11 @@ export abstract class BaseNode { } if (this.kNode instanceof Konva.Group) { const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect; - wrapperRect.strokeWidth(0); + console.log("Wrapper rect:", wrapperRect); + if (wrapperRect !=null) { + // Issue investigate + wrapperRect.strokeWidth(0); + } return; } if (import.meta.env.DEV) { diff --git a/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts index 9b2685ecf7..0db9c4bc14 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts @@ -83,8 +83,10 @@ export class ImageNode extends NetworkedNode { }) { const imageComponent = new Image(); imageComponent.crossOrigin = "anonymous"; + // this.setProgress({ progress: 100, status: UploadStatus.SUCCESS }); + imageComponent.onload = () => { - this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED }); + //this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED }); this.imageSize = { width: imageComponent.width, height: imageComponent.height, @@ -105,33 +107,33 @@ export class ImageNode extends NetworkedNode { this.kNode.fill(transparent); this.listenToBaseKNode(); this.mediaLayerRef.draw(); - this.uploadImage(imageFile); + //this.uploadImage(imageFile); }; imageComponent.onerror = () => { - this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE }); + //this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE }); }; imageComponent.src = URL.createObjectURL(imageFile); } private async loadImageFromUrl(mediaFileUrl: string) { - this.setProgress({ progress: 75, status: UploadStatus.LOADING }); + //this.setProgress({ progress: 75, status: UploadStatus.LOADING }); const newImage = new Image(); newImage.crossOrigin = "anonymous"; newImage.onerror = () => { - this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD }); + //this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD }); }; newImage.onload = () => { // console.log("network image", newImage); this.kNode.image(newImage); this.kNode.draw(); - this.setProgress({ progress: 100, status: UploadStatus.SUCCESS }); + //this.setProgress({ progress: 100, status: UploadStatus.SUCCESS }); }; newImage.src = mediaFileUrl; this.listenToBaseKNode(); } private async uploadImage(imageFile: File) { - this.setProgress({ progress: 10, status: UploadStatus.UPLOADING }); + //this.setProgress({ progress: 10, status: UploadStatus.UPLOADING }); const mediaUploadApi = new MediaUploadApi(); const uploadResponse = await mediaUploadApi.UploadImage({ @@ -149,7 +151,7 @@ export class ImageNode extends NetworkedNode { } private async retreiveImage(mediaFileToken: string) { - this.setProgress({ progress: 50, status: UploadStatus.RETREIVING }); + //this.setProgress({ progress: 50, status: UploadStatus.RETREIVING }); const mediaFileApi = new MediaFilesApi(); const mediaFileResponse = await mediaFileApi.GetMediaFileByToken({ mediaFileToken: mediaFileToken, diff --git a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts new file mode 100644 index 0000000000..6e0b7520c5 --- /dev/null +++ b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts @@ -0,0 +1,194 @@ +import Konva from "konva"; +import { SelectionManager } from "../NodesManagers"; +import { Position, Size, NodeData, TransformationData } from "../types"; +import { BaseNode } from "./BaseNode"; +import { NodeType } from "./constants"; +import { NodeUtilities } from "./NodeUtilities"; + +export enum ShapeType { + CIRCLE = "circle", + SQUARE = "square", + TRIANGLE = "triangle", +} + +interface ShapeNodeConstructor { + canvasPosition: Position; + canvasSize: Size; + shapeType: ShapeType; + size: number; + color?: string; // Hex color string + transform?: TransformationData; + mediaLayerRef: Konva.Layer; + selectionManagerRef: SelectionManager; +} + +export class ShapeNode extends BaseNode { + public kNode: Konva.Group; + + constructor({ + canvasPosition, + canvasSize, + shapeType, + size, + color = "#ff0000", // Default red if no color provided + transform: existingTransform, + mediaLayerRef, + selectionManagerRef, + }: ShapeNodeConstructor) { + // Create the actual shape inside the group + const transform = NodeUtilities.getInitialTransform({ + existingTransform, + canvasPosition, + canvasSize, + }); + let shape = null; + switch (shapeType) { + case ShapeType.CIRCLE: + const circleShape = new Konva.Circle({ + radius: 50, + ...transform, + fill: color, + strokeScaleEnabled: false, + draggable: true, + }); + + // Convert circle to image + const dataURL = circleShape.toDataURL(); + const imageObj = new Image(); + imageObj.src = dataURL; + + shape = new Konva.Image({ + image: imageObj, + x: circleShape.x(), + y: circleShape.y(), + width: circleShape.width(), + height: circleShape.height(), + ...transform, + fill: 'transparent', + strokeScaleEnabled: false, + draggable: true, + }); + + imageObj.onload = () => { + this.mediaLayerRef.draw(); + }; + + // Clean up the temporary circle + circleShape.destroy(); + break; + + case ShapeType.SQUARE: + const squareShape = new Konva.Rect({ + width: 100, + height: 100, + ...transform, + fill: color, + strokeScaleEnabled: false, + draggable: true, + }); + + // Convert square to image + const squareDataURL = squareShape.toDataURL(); + const squareImageObj = new Image(); + squareImageObj.src = squareDataURL; + + shape = new Konva.Image({ + image: squareImageObj, + x: squareShape.x(), + y: squareShape.y(), + width: squareShape.width(), + height: squareShape.height(), + ...transform, + fill: 'transparent', + strokeScaleEnabled: false, + draggable: true, + }); + + squareImageObj.onload = () => { + this.mediaLayerRef.draw(); + }; + + // Clean up the temporary square + squareShape.destroy(); + break; + + case ShapeType.TRIANGLE: + const triangleShape = new Konva.RegularPolygon({ + sides: 3, + radius: 50, + ...transform, + fill: color, + strokeScaleEnabled: false, + draggable: true, + }); + + // Convert triangle to image + const triangleDataURL = triangleShape.toDataURL(); + const triangleImageObj = new Image(); + triangleImageObj.src = triangleDataURL; + + shape = new Konva.Image({ + image: triangleImageObj, + x: triangleShape.x(), + y: triangleShape.y(), + width: triangleShape.width(), + height: triangleShape.height(), + ...transform, + fill: 'transparent', + strokeScaleEnabled: false, + draggable: true, + }); + + triangleImageObj.onload = () => { + this.mediaLayerRef.draw(); + }; + + // Clean up the temporary triangle + triangleShape.destroy(); + break; + + default: + throw new Error("Invalid shape type"); + } + + super({ + selectionManagerRef: selectionManagerRef, + mediaLayerRef: mediaLayerRef, + kNode: shape, + }); + + this.kNode = shape; + this.shapeType = shapeType; + + // Add shape to group + const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({ + canvasOffset: canvasPosition, + componentSize: canvasSize, + maxSize: canvasSize, + }); + this.kNode.setPosition(centerPosition); + this.mediaLayerRef.add(this.kNode); + this.listenToBaseKNode(); + this.mediaLayerRef.draw(); + } + + public getNodeData(canvasPosition: Position) { + const data: NodeData = { + type: NodeType.SHAPE, + transform: { + position: { + x: this.kNode.position().x - canvasPosition.x, + y: this.kNode.position().y - canvasPosition.y, + }, + size: this.kNode.size(), + rotation: this.kNode.rotation(), + scale: { + x: this.kNode.scaleX(), + y: this.kNode.scaleY(), + }, + zIndex: this.kNode.getZIndex(), + }, + }; + return data; + } +} diff --git a/frontend/realtime/src/KonvaApp/Nodes/constants.ts b/frontend/realtime/src/KonvaApp/Nodes/constants.ts index d5a236701d..e6f46020ae 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/constants.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/constants.ts @@ -4,6 +4,7 @@ export enum NodeType { IMAGE = "image", TEXT = "text", VIDEO = "video", + SHAPE = "shape" } export const transparent = Colors.transparent; export const primaryOrange = Colors.primaryOrange; diff --git a/frontend/realtime/src/KonvaApp/Nodes/index.ts b/frontend/realtime/src/KonvaApp/Nodes/index.ts index 955127087e..7c4b9a4386 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/index.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/index.ts @@ -3,3 +3,4 @@ export { BaseNode } from "./BaseNode"; export { ImageNode } from "./ImageNode"; export { VideoNode } from "./VideoNode"; export { TextNode } from "./TextNode"; +export { ShapeNode } from "./ShapeNode"; \ No newline at end of file diff --git a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts index 44a6c2dded..e5c69474d1 100644 --- a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts +++ b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts @@ -3,22 +3,24 @@ import { Container } from "konva/lib/Container"; import { Shape } from "konva/lib/Shape"; import { Group } from "konva/lib/Group"; - import { invoke } from "@tauri-apps/api/core"; - import { FileUtilities } from "../FileUtilities/FileUtilities"; -import { ImageNode, VideoNode, TextNode } from "../Nodes"; +import { ImageNode, VideoNode, TextNode, ShapeNode } from "../Nodes"; import { MediaNode } from "../types"; import { RenderTask } from "./RenderTask"; import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas"; // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio - +export enum ShapeType { + Square = "square", + Triangle = "triangle", + Circle = "circle" +} export class RealTimeDrawEngine { private videoNodes: VideoNode[]; - private imageNodes: (ImageNode | TextNode)[]; + private imageNodes: (ImageNode | TextNode | ShapeNode)[]; private offScreenCanvas: OffscreenCanvas; private outputBitmap: ImageBitmap | undefined; @@ -26,7 +28,7 @@ export class RealTimeDrawEngine { // private frames: ImageBitmap[]; // capturing composite within window - private bgLayerRef: Konva.Layer; + private mediaLayerRef: Konva.Layer; private height: number; @@ -51,13 +53,11 @@ export class RealTimeDrawEngine { constructor({ width, height, - bgLayerRef, mediaLayerRef, offScreenCanvas, }: { width: number; height: number; - bgLayerRef: Konva.Layer; mediaLayerRef: Konva.Layer; offScreenCanvas: OffscreenCanvas; }) { @@ -80,7 +80,6 @@ export class RealTimeDrawEngine { this.offScreenCanvas.width = this.width; this.offScreenCanvas.height = this.height; - this.bgLayerRef = bgLayerRef; // this is the whole canvas this.mediaLayerRef = mediaLayerRef; @@ -90,7 +89,7 @@ export class RealTimeDrawEngine { this.port = undefined; this.fps = 24; - + this.currentPrompt = ""; this.currentStrength = 100; // This is captures a subset of the medialayer ref @@ -106,20 +105,6 @@ export class RealTimeDrawEngine { draggable: false, }); - // Add preview canvas with same dimensions but different style - // this.previewCanvas = new Konva.Rect({ - // name: "PreviewCanvas", - // x: this.positionX, - // y: this.positionY, - // width: this.width, - // height: this.height, - // fill: "white", - // stroke: "blue", - // strokeWidth: 1, - // dash: [5, 5], - // draggable: false, - // }); - this.previewCanvas = new Konva.Image({ name: "PreviewCanvas", x: this.positionX, @@ -127,19 +112,17 @@ export class RealTimeDrawEngine { width: this.width, height: this.height, image: undefined, - stroke: "blue", + stroke: "black", strokeWidth: 1, - dash: [5, 5], draggable: false, + fill: "white", }); - this.bgLayerRef.add(this.captureCanvas); - this.bgLayerRef.add(this.previewCanvas); + this.mediaLayerRef.add(this.captureCanvas); + this.mediaLayerRef.add(this.previewCanvas); // send back this.captureCanvas.setZIndex(0); - this.previewCanvas.setZIndex(0); - - //this.debug(); + this.previewCanvas.setZIndex(1); } async updateCaptureCanvas( @@ -284,12 +267,13 @@ export class RealTimeDrawEngine { this.isProcessing = true; await this.render(); }; + public addNodes(node: MediaNode) { - if (node instanceof VideoNode) { - this.videoNodes.push(node); - node.kNode.on("dragend", this.handleNodeDragEnd); - } else if (node instanceof ImageNode || node instanceof TextNode) { + + if (node instanceof ImageNode || node instanceof TextNode || node instanceof ShapeNode) { + console.debug("Adding node:", node); this.imageNodes.push(node); + console.log(this.imageNodes) node.kNode.on("dragend", this.handleNodeDragEnd); } } @@ -325,6 +309,7 @@ export class RealTimeDrawEngine { try { const box = config.layerOfInterest.getClientRect(); const stage = config.layerOfInterest.getStage(); + const x = config.x !== undefined ? config.x : Math.floor(box.x); const y = config.y !== undefined ? config.y : Math.floor(box.y); @@ -384,8 +369,6 @@ export class RealTimeDrawEngine { } } - - private async imageBitmapToBase64(imageBitmap: ImageBitmap): Promise { // Create a temporary canvas const canvas = document.createElement("canvas"); @@ -462,9 +445,11 @@ export class RealTimeDrawEngine { } public async render() { - // only pick nodes that intersect with the canvas on screen bounds to freeze. + // only pick nodes that intersect wi th the canvas on screen bounds to freeze. this.mediaLayerRef.draw(); - + // Output all nodes in mediaLayerRef + const nodes = this.mediaLayerRef.getChildren(); + console.log("All nodes in mediaLayer:", nodes); console.log( `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`, ); @@ -482,20 +467,20 @@ export class RealTimeDrawEngine { })) as ImageBitmap; // Test code - if (false) { + if (true) { this.outputBitmap = bitmap; this.previewCanvas.image(bitmap); this.isProcessing = false; - return - } + return; + } try { const base64Bitmap = await this.imageBitmapToBase64(bitmap); const base64BitmapResponse = await invoke("infer_image", { image: base64Bitmap, - prompt:this.currentPrompt, - strength:this.currentStrength, + prompt: this.currentPrompt, + strength: this.currentStrength, }); console.log(base64BitmapResponse); @@ -510,6 +495,5 @@ export class RealTimeDrawEngine { } finally { this.isProcessing = false; } - } } diff --git a/frontend/realtime/src/KonvaApp/types/NodeData.ts b/frontend/realtime/src/KonvaApp/types/NodeData.ts index 2cc55ee0c2..dae440e821 100644 --- a/frontend/realtime/src/KonvaApp/types/NodeData.ts +++ b/frontend/realtime/src/KonvaApp/types/NodeData.ts @@ -1,4 +1,4 @@ -import { NodeType, BaseNode, ImageNode, TextNode, VideoNode } from "../Nodes"; +import { NodeType, BaseNode, ImageNode, TextNode, VideoNode, ShapeNode } from "../Nodes"; import { TransformationData } from "./Transformation"; @@ -6,7 +6,7 @@ import { TextNodeData } from "./Text"; export type MediaNode = // | NetworkedNode - BaseNode | VideoNode | ImageNode | TextNode; + BaseNode | VideoNode | ImageNode | TextNode | ShapeNode; export type ImageNodeData = { mediaFileUrl: string; diff --git a/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx b/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx index cf03008333..2222c92286 100644 --- a/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx +++ b/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx @@ -42,12 +42,6 @@ export const DialogAddImage = ({ closeCallback(); } - function handleEnter() { - if (currFile) { - onAddImage(currFile); - } - handleClose(); - } return (
@@ -66,6 +60,10 @@ export const DialogAddImage = ({ file={currFile} setFile={(file: File | null) => { setAssetFile(file); + if (file) { + onAddImage(file); + handleClose(); + } }} /> {currFile && ( @@ -84,9 +82,6 @@ export const DialogAddImage = ({ -
diff --git a/frontend/realtime/src/signals/uiEvents/addMedia.ts b/frontend/realtime/src/signals/uiEvents/addMedia.ts index 8b85ff5bff..c546061025 100644 --- a/frontend/realtime/src/signals/uiEvents/addMedia.ts +++ b/frontend/realtime/src/signals/uiEvents/addMedia.ts @@ -6,6 +6,7 @@ const stagedImage = signal(null); const addImageToEngine = (image: File) => { stagedImage.value = image; }; + const onGetStagedImage = (callback: (file: File) => void) => { effect(() => { if (stagedImage.value) {