From 9147da22f2273a77bca5b29d1805ce864e8511a5 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 19 Feb 2025 00:09:47 -0500 Subject: [PATCH 1/4] ShapeNodes work need to fix transformations. --- frontend/realtime/src/KonvaApp/Engine.ts | 21 ++- .../EngineUtitlities/CommandManager.ts | 4 +- .../realtime/src/KonvaApp/Nodes/BaseNode.ts | 13 +- .../realtime/src/KonvaApp/Nodes/ShapeNode.ts | 126 ++++++++++++++++++ .../realtime/src/KonvaApp/Nodes/constants.ts | 1 + frontend/realtime/src/KonvaApp/Nodes/index.ts | 1 + .../RenderingPrimitives/RealTimeDrawEngine.ts | 54 +++----- .../realtime/src/KonvaApp/types/NodeData.ts | 4 +- 8 files changed, 181 insertions(+), 43 deletions(-) create mode 100644 frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts diff --git a/frontend/realtime/src/KonvaApp/Engine.ts b/frontend/realtime/src/KonvaApp/Engine.ts index d25ad93e8b..eb07aeb38d 100644 --- a/frontend/realtime/src/KonvaApp/Engine.ts +++ b/frontend/realtime/src/KonvaApp/Engine.ts @@ -2,6 +2,7 @@ import Konva from "konva"; import { uiAccess, uiEvents } from "~/signals"; +import { ShapeNode } from "./Nodes"; import { UndoStackManager } from "./UndoRedo"; import { CommandManager, MatteBox, SceneManager } from "./EngineUtitlities"; @@ -26,7 +27,7 @@ import { NavigateFunction } from "react-router-dom"; import { LoadingVideosProvider } from "./EngineUtitlities/LoadingVideosProvider"; import { VideoExtractionHandler } from "./EngineUtitlities/VideoExtractionHandler/VideoExtractionHandler"; -import { RealTimeDrawEngine } from "./RenderingPrimitives/RealTimeDrawEngine"; +import { RealTimeDrawEngine, ShapeType } from "./RenderingPrimitives/RealTimeDrawEngine"; // for testing loading files from system // import { FileUtilities } from "./FileUtilities/FileUtilities"; @@ -393,6 +394,10 @@ export class Engine { this.addImage(image); }); + + this.addShape(ShapeType.Circle,100); + this.addShape(ShapeType.Square,100); + this.addShape(ShapeType.Triangle,100); uiEvents.onAddTextToEngine((textdata) => { this.addText(textdata); @@ -525,6 +530,20 @@ export class Engine { this.commandManager.createNode(textNode); } + public addShape(type: ShapeType, size: number, color?: string) { + const shapeNode = new ShapeNode({ + canvasPosition: this.realTimeDrawEngine.captureCanvas.position(), + canvasSize: this.realTimeDrawEngine.captureCanvas.size(), + shapeType: type, + size: size, + color: color, + mediaLayerRef: this.mediaLayer, + selectionManagerRef: this.selectionManager + }); + this.realTimeDrawEngine.addNodes(shapeNode); + this.commandManager.createNode(shapeNode); + } + public addImage(imageFile: File) { const imageNode = new ImageNode({ mediaLayerRef: this.mediaLayer, diff --git a/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts b/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts index cc4862f536..bb008055f2 100644 --- a/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts +++ b/frontend/realtime/src/KonvaApp/EngineUtitlities/CommandManager.ts @@ -1,5 +1,5 @@ import Konva from "konva"; -import { ImageNode, TextNode, VideoNode } from "../Nodes"; +import { ImageNode, ShapeNode, TextNode, VideoNode } from "../Nodes"; import { MediaNode, Position, RGBColor, Transformation } from "../types"; import { AddChromaKey, @@ -48,7 +48,7 @@ export class CommandManager { this.undoStackManagerRef = engineRefs.undoStackManagerRef; } - createNode(node: VideoNode | ImageNode | TextNode) { + createNode(node: VideoNode | ImageNode | TextNode | ShapeNode) { const command = new CreateCommand({ nodes: new Set([node]), mediaLayerRef: this.mediaLayerRef, diff --git a/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts b/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts index f9e60a9983..de47252474 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts @@ -67,8 +67,11 @@ export abstract class BaseNode { } if (this.kNode instanceof Konva.Group) { const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect; - wrapperRect.stroke(primaryOrange); - wrapperRect.strokeWidth(highlightStrokeWidth); + console.log("Wrapper rect:", wrapperRect); + if (wrapperRect !=null) { + wrapperRect.stroke(primaryOrange); + wrapperRect.strokeWidth(highlightStrokeWidth); + } return; } if (import.meta.env.DEV) { @@ -86,7 +89,11 @@ export abstract class BaseNode { } if (this.kNode instanceof Konva.Group) { const wrapperRect = this.kNode.findOne(".wrapper") as Konva.Rect; - wrapperRect.strokeWidth(0); + console.log("Wrapper rect:", wrapperRect); + if (wrapperRect !=null) { + // Issue investigate + wrapperRect.strokeWidth(0); + } return; } if (import.meta.env.DEV) { diff --git a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts new file mode 100644 index 0000000000..1a6c12b282 --- /dev/null +++ b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts @@ -0,0 +1,126 @@ +import Konva from "konva"; +import { SelectionManager } from "../NodesManagers"; +import { Position, Size, NodeData, TransformationData } from "../types"; +import { BaseNode } from "./BaseNode"; +import { NodeType } from "./constants"; +import { NodeUtilities } from "./NodeUtilities"; + +export enum ShapeType { + CIRCLE = "circle", + SQUARE = "square", + TRIANGLE = "triangle" + } + +interface ShapeNodeConstructor { + canvasPosition: Position; + canvasSize: Size; + shapeType: ShapeType; + size: number; + color?: string; // Hex color string + transform?: TransformationData; + mediaLayerRef: Konva.Layer; + selectionManagerRef: SelectionManager; +} + +export class ShapeNode extends BaseNode { + public kNode: Konva.Group; + private shapeType: ShapeType; + private shape: Konva.Shape; + + constructor({ + canvasPosition, + canvasSize, + shapeType, + size, + color = "#ff0000", // Default red if no color provided + transform: existingTransform, + mediaLayerRef, + selectionManagerRef, + }: ShapeNodeConstructor) { + + const transform = NodeUtilities.getInitialTransform({ + existingTransform, + canvasPosition, + canvasSize, + }); + + // Create group to satisfy BaseNode type requirements + const group = new Konva.Group({ + ...transform, + draggable: true, + }); + + super({ + selectionManagerRef: selectionManagerRef, + mediaLayerRef: mediaLayerRef, + kNode: group + }); + + this.kNode = group; + this.shapeType = shapeType; + + // Create the actual shape inside the group + switch (shapeType) { + case ShapeType.CIRCLE: + this.shape = new Konva.Circle({ + radius: 50, + fill: color, + strokeScaleEnabled: false + }); + break; + + case ShapeType.SQUARE: + this.shape = new Konva.Rect({ + width: 100, + height: 100, + fill: color, + strokeScaleEnabled: false + }); + break; + + case ShapeType.TRIANGLE: + this.shape = new Konva.RegularPolygon({ + sides: 3, + radius: 50, + fill: color, + strokeScaleEnabled: false + }); + break; + + default: + throw new Error('Invalid shape type'); + } + + // Add shape to group + const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({ + canvasOffset: canvasPosition, + componentSize: canvasSize, + maxSize: canvasSize, + }); + this.kNode.setPosition(centerPosition); + this.kNode.add(this.shape); + this.mediaLayerRef.add(this.kNode); + this.listenToBaseKNode(); + this.mediaLayerRef.draw(); + } + + public getNodeData(canvasPosition: Position) { + const data: NodeData = { + type: NodeType.SHAPE, + transform: { + position: { + x: this.kNode.position().x - canvasPosition.x, + y: this.kNode.position().y - canvasPosition.y, + }, + size: this.kNode.size(), + rotation: this.kNode.rotation(), + scale: { + x: this.kNode.scaleX(), + y: this.kNode.scaleY(), + }, + zIndex: this.kNode.getZIndex(), + } + }; + return data; + } +} diff --git a/frontend/realtime/src/KonvaApp/Nodes/constants.ts b/frontend/realtime/src/KonvaApp/Nodes/constants.ts index d5a236701d..e6f46020ae 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/constants.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/constants.ts @@ -4,6 +4,7 @@ export enum NodeType { IMAGE = "image", TEXT = "text", VIDEO = "video", + SHAPE = "shape" } export const transparent = Colors.transparent; export const primaryOrange = Colors.primaryOrange; diff --git a/frontend/realtime/src/KonvaApp/Nodes/index.ts b/frontend/realtime/src/KonvaApp/Nodes/index.ts index 955127087e..7c4b9a4386 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/index.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/index.ts @@ -3,3 +3,4 @@ export { BaseNode } from "./BaseNode"; export { ImageNode } from "./ImageNode"; export { VideoNode } from "./VideoNode"; export { TextNode } from "./TextNode"; +export { ShapeNode } from "./ShapeNode"; \ No newline at end of file diff --git a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts index 44a6c2dded..2c3f61521a 100644 --- a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts +++ b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts @@ -3,22 +3,24 @@ import { Container } from "konva/lib/Container"; import { Shape } from "konva/lib/Shape"; import { Group } from "konva/lib/Group"; - import { invoke } from "@tauri-apps/api/core"; - import { FileUtilities } from "../FileUtilities/FileUtilities"; -import { ImageNode, VideoNode, TextNode } from "../Nodes"; +import { ImageNode, VideoNode, TextNode, ShapeNode } from "../Nodes"; import { MediaNode } from "../types"; import { RenderTask } from "./RenderTask"; import { OffScreenSceneCanvas } from "./OffScreenSceneCanvas"; // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio - +export enum ShapeType { + Square = "square", + Triangle = "triangle", + Circle = "circle" +} export class RealTimeDrawEngine { private videoNodes: VideoNode[]; - private imageNodes: (ImageNode | TextNode)[]; + private imageNodes: (ImageNode | TextNode | ShapeNode)[]; private offScreenCanvas: OffscreenCanvas; private outputBitmap: ImageBitmap | undefined; @@ -90,7 +92,7 @@ export class RealTimeDrawEngine { this.port = undefined; this.fps = 24; - + this.currentPrompt = ""; this.currentStrength = 100; // This is captures a subset of the medialayer ref @@ -100,26 +102,12 @@ export class RealTimeDrawEngine { y: this.positionY, width: this.width, height: this.height, - fill: "white", + fill: "blue", stroke: "black", strokeWidth: 1, draggable: false, }); - // Add preview canvas with same dimensions but different style - // this.previewCanvas = new Konva.Rect({ - // name: "PreviewCanvas", - // x: this.positionX, - // y: this.positionY, - // width: this.width, - // height: this.height, - // fill: "white", - // stroke: "blue", - // strokeWidth: 1, - // dash: [5, 5], - // draggable: false, - // }); - this.previewCanvas = new Konva.Image({ name: "PreviewCanvas", x: this.positionX, @@ -127,16 +115,16 @@ export class RealTimeDrawEngine { width: this.width, height: this.height, image: undefined, - stroke: "blue", + stroke: "black", strokeWidth: 1, - dash: [5, 5], draggable: false, + fill: "red", }); this.bgLayerRef.add(this.captureCanvas); this.bgLayerRef.add(this.previewCanvas); // send back - this.captureCanvas.setZIndex(0); + this.captureCanvas.setZIndex(1); this.previewCanvas.setZIndex(0); //this.debug(); @@ -284,11 +272,10 @@ export class RealTimeDrawEngine { this.isProcessing = true; await this.render(); }; + public addNodes(node: MediaNode) { - if (node instanceof VideoNode) { - this.videoNodes.push(node); - node.kNode.on("dragend", this.handleNodeDragEnd); - } else if (node instanceof ImageNode || node instanceof TextNode) { + + if (node instanceof ImageNode || node instanceof TextNode || node instanceof ShapeNode) { this.imageNodes.push(node); node.kNode.on("dragend", this.handleNodeDragEnd); } @@ -384,8 +371,6 @@ export class RealTimeDrawEngine { } } - - private async imageBitmapToBase64(imageBitmap: ImageBitmap): Promise { // Create a temporary canvas const canvas = document.createElement("canvas"); @@ -486,16 +471,16 @@ export class RealTimeDrawEngine { this.outputBitmap = bitmap; this.previewCanvas.image(bitmap); this.isProcessing = false; - return - } + return; + } try { const base64Bitmap = await this.imageBitmapToBase64(bitmap); const base64BitmapResponse = await invoke("infer_image", { image: base64Bitmap, - prompt:this.currentPrompt, - strength:this.currentStrength, + prompt: this.currentPrompt, + strength: this.currentStrength, }); console.log(base64BitmapResponse); @@ -510,6 +495,5 @@ export class RealTimeDrawEngine { } finally { this.isProcessing = false; } - } } diff --git a/frontend/realtime/src/KonvaApp/types/NodeData.ts b/frontend/realtime/src/KonvaApp/types/NodeData.ts index 2cc55ee0c2..dae440e821 100644 --- a/frontend/realtime/src/KonvaApp/types/NodeData.ts +++ b/frontend/realtime/src/KonvaApp/types/NodeData.ts @@ -1,4 +1,4 @@ -import { NodeType, BaseNode, ImageNode, TextNode, VideoNode } from "../Nodes"; +import { NodeType, BaseNode, ImageNode, TextNode, VideoNode, ShapeNode } from "../Nodes"; import { TransformationData } from "./Transformation"; @@ -6,7 +6,7 @@ import { TextNodeData } from "./Text"; export type MediaNode = // | NetworkedNode - BaseNode | VideoNode | ImageNode | TextNode; + BaseNode | VideoNode | ImageNode | TextNode | ShapeNode; export type ImageNodeData = { mediaFileUrl: string; From 442c3579e11eb2f001ef28a3fc526deec95533a3 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 19 Feb 2025 17:59:14 -0500 Subject: [PATCH 2/4] Fixed Modal to have the image enter the scene right away. Added Shape Fixes resizes properly and drags. Removing Progress Fill. --- .../realtime/src/KonvaApp/Nodes/BaseNode.ts | 2 +- .../realtime/src/KonvaApp/Nodes/ImageNode.ts | 18 +++-- .../realtime/src/KonvaApp/Nodes/ShapeNode.ts | 76 ++++++++++--------- .../DialogAddImage/DialogAddImage.tsx | 13 +--- .../realtime/src/signals/uiEvents/addMedia.ts | 1 + 5 files changed, 56 insertions(+), 54 deletions(-) diff --git a/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts b/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts index de47252474..1ca2cd8c83 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/BaseNode.ts @@ -29,7 +29,7 @@ export abstract class BaseNode { }: { selectionManagerRef: SelectionManager; mediaLayerRef: Konva.Layer; - kNode: Konva.Image | Konva.Group; + kNode: Konva.Image | Konva.Group | Konva.Shape; }) { // console.log("Node constructed"); // this.uuid = uuidv4(); diff --git a/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts index 9b2685ecf7..0db9c4bc14 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/ImageNode.ts @@ -83,8 +83,10 @@ export class ImageNode extends NetworkedNode { }) { const imageComponent = new Image(); imageComponent.crossOrigin = "anonymous"; + // this.setProgress({ progress: 100, status: UploadStatus.SUCCESS }); + imageComponent.onload = () => { - this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED }); + //this.setProgress({ progress: 0, status: UploadStatus.FILE_STAGED }); this.imageSize = { width: imageComponent.width, height: imageComponent.height, @@ -105,33 +107,33 @@ export class ImageNode extends NetworkedNode { this.kNode.fill(transparent); this.listenToBaseKNode(); this.mediaLayerRef.draw(); - this.uploadImage(imageFile); + //this.uploadImage(imageFile); }; imageComponent.onerror = () => { - this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE }); + //this.setProgress({ progress: 0, status: UploadStatus.ERROR_ON_FILE }); }; imageComponent.src = URL.createObjectURL(imageFile); } private async loadImageFromUrl(mediaFileUrl: string) { - this.setProgress({ progress: 75, status: UploadStatus.LOADING }); + //this.setProgress({ progress: 75, status: UploadStatus.LOADING }); const newImage = new Image(); newImage.crossOrigin = "anonymous"; newImage.onerror = () => { - this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD }); + //this.setProgress({ progress: 90, status: UploadStatus.ERROR_ON_LOAD }); }; newImage.onload = () => { // console.log("network image", newImage); this.kNode.image(newImage); this.kNode.draw(); - this.setProgress({ progress: 100, status: UploadStatus.SUCCESS }); + //this.setProgress({ progress: 100, status: UploadStatus.SUCCESS }); }; newImage.src = mediaFileUrl; this.listenToBaseKNode(); } private async uploadImage(imageFile: File) { - this.setProgress({ progress: 10, status: UploadStatus.UPLOADING }); + //this.setProgress({ progress: 10, status: UploadStatus.UPLOADING }); const mediaUploadApi = new MediaUploadApi(); const uploadResponse = await mediaUploadApi.UploadImage({ @@ -149,7 +151,7 @@ export class ImageNode extends NetworkedNode { } private async retreiveImage(mediaFileToken: string) { - this.setProgress({ progress: 50, status: UploadStatus.RETREIVING }); + //this.setProgress({ progress: 50, status: UploadStatus.RETREIVING }); const mediaFileApi = new MediaFilesApi(); const mediaFileResponse = await mediaFileApi.GetMediaFileByToken({ mediaFileToken: mediaFileToken, diff --git a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts index 1a6c12b282..df4e3446a7 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts @@ -6,11 +6,11 @@ import { NodeType } from "./constants"; import { NodeUtilities } from "./NodeUtilities"; export enum ShapeType { - CIRCLE = "circle", - SQUARE = "square", - TRIANGLE = "triangle" - } - + CIRCLE = "circle", + SQUARE = "square", + TRIANGLE = "triangle", +} + interface ShapeNodeConstructor { canvasPosition: Position; canvasSize: Size; @@ -37,68 +37,72 @@ export class ShapeNode extends BaseNode { mediaLayerRef, selectionManagerRef, }: ShapeNodeConstructor) { - + // Create the actual shape inside the group const transform = NodeUtilities.getInitialTransform({ existingTransform, canvasPosition, canvasSize, }); - - // Create group to satisfy BaseNode type requirements - const group = new Konva.Group({ - ...transform, - draggable: true, - }); - - super({ - selectionManagerRef: selectionManagerRef, - mediaLayerRef: mediaLayerRef, - kNode: group - }); - - this.kNode = group; - this.shapeType = shapeType; - - // Create the actual shape inside the group + let shape = null; switch (shapeType) { case ShapeType.CIRCLE: - this.shape = new Konva.Circle({ + shape = new Konva.Circle({ radius: 50, fill: color, - strokeScaleEnabled: false + ...transform, + strokeScaleEnabled: false, + draggable: true, }); break; case ShapeType.SQUARE: - this.shape = new Konva.Rect({ + shape = new Konva.Rect({ width: 100, height: 100, + ...transform, fill: color, - strokeScaleEnabled: false + strokeScaleEnabled: false, + draggable: true, }); break; case ShapeType.TRIANGLE: - this.shape = new Konva.RegularPolygon({ + shape = new Konva.RegularPolygon({ sides: 3, radius: 50, + ...transform, fill: color, - strokeScaleEnabled: false + strokeScaleEnabled: false, + draggable: true, }); break; default: - throw new Error('Invalid shape type'); + throw new Error("Invalid shape type"); } + // Create group to satisfy BaseNode type requirements + // const group = new Konva.Group({ + // ...transform, + // draggable: true, + // }); + + super({ + selectionManagerRef: selectionManagerRef, + mediaLayerRef: mediaLayerRef, + kNode: shape, + }); + + this.kNode = shape; + this.shapeType = shapeType; + // Add shape to group const centerPosition = NodeUtilities.positionNodeOnCanvasCenter({ - canvasOffset: canvasPosition, - componentSize: canvasSize, - maxSize: canvasSize, - }); + canvasOffset: canvasPosition, + componentSize: canvasSize, + maxSize: canvasSize, + }); this.kNode.setPosition(centerPosition); - this.kNode.add(this.shape); this.mediaLayerRef.add(this.kNode); this.listenToBaseKNode(); this.mediaLayerRef.draw(); @@ -119,7 +123,7 @@ export class ShapeNode extends BaseNode { y: this.kNode.scaleY(), }, zIndex: this.kNode.getZIndex(), - } + }, }; return data; } diff --git a/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx b/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx index cf03008333..2222c92286 100644 --- a/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx +++ b/frontend/realtime/src/components/features/DialogAddImage/DialogAddImage.tsx @@ -42,12 +42,6 @@ export const DialogAddImage = ({ closeCallback(); } - function handleEnter() { - if (currFile) { - onAddImage(currFile); - } - handleClose(); - } return (
@@ -66,6 +60,10 @@ export const DialogAddImage = ({ file={currFile} setFile={(file: File | null) => { setAssetFile(file); + if (file) { + onAddImage(file); + handleClose(); + } }} /> {currFile && ( @@ -84,9 +82,6 @@ export const DialogAddImage = ({ -
diff --git a/frontend/realtime/src/signals/uiEvents/addMedia.ts b/frontend/realtime/src/signals/uiEvents/addMedia.ts index 1b1f29a8be..e2d09e2757 100644 --- a/frontend/realtime/src/signals/uiEvents/addMedia.ts +++ b/frontend/realtime/src/signals/uiEvents/addMedia.ts @@ -6,6 +6,7 @@ const stagedImage = signal(null); const addImageToEngine = (image: File) => { stagedImage.value = image; }; + const onGetStagedImage = (callback: (file: File) => void) => { effect(() => { if (stagedImage.value) { From 6e03df968df827ea3170912516b3c487777d3f63 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 19 Feb 2025 19:34:06 -0500 Subject: [PATCH 3/4] Remove duplicate adds? Then fixed layering issue by removing useless background layer. Need to fix shape rendering. --- frontend/realtime/src/KonvaApp/Engine.ts | 20 ++++++++++------ .../realtime/src/KonvaApp/Nodes/ShapeNode.ts | 10 +------- .../RenderingPrimitives/RealTimeDrawEngine.ts | 23 ++++++++----------- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/frontend/realtime/src/KonvaApp/Engine.ts b/frontend/realtime/src/KonvaApp/Engine.ts index eb07aeb38d..fac6ddd5f5 100644 --- a/frontend/realtime/src/KonvaApp/Engine.ts +++ b/frontend/realtime/src/KonvaApp/Engine.ts @@ -98,6 +98,7 @@ export class Engine { this.stage.add(this.nodeIsolationLayer); this.stage.add(this.uiLayer); + // Konva Transformer this.nodeTransformer = new NodeTransformer(); this.uiLayer.add(this.nodeTransformer.getKonvaNode()); @@ -178,6 +179,12 @@ export class Engine { // hence, lastly, setup these events this.setupEventSystem(); this.setAppMode(AppModes.SELECT); + + + console.log("HELLO") + this.addShape(ShapeType.Circle,100); + // this.addShape(ShapeType.Square,100); + // this.addShape(ShapeType.Triangle,100); } @@ -394,10 +401,7 @@ export class Engine { this.addImage(image); }); - - this.addShape(ShapeType.Circle,100); - this.addShape(ShapeType.Square,100); - this.addShape(ShapeType.Triangle,100); + uiEvents.onAddTextToEngine((textdata) => { this.addText(textdata); @@ -540,8 +544,10 @@ export class Engine { mediaLayerRef: this.mediaLayer, selectionManagerRef: this.selectionManager }); - this.realTimeDrawEngine.addNodes(shapeNode); + shapeNode.kNode.zIndex(1); // Replace desiredZIndex with a number + this.commandManager.createNode(shapeNode); + //this.realTimeDrawEngine.addNodes(shapeNode); } public addImage(imageFile: File) { @@ -552,9 +558,9 @@ export class Engine { imageFile: imageFile, selectionManagerRef: this.selectionManager, }); - + imageNode.kNode.zIndex(1); this.commandManager.createNode(imageNode); - this.realTimeDrawEngine.addNodes(imageNode); + //this.realTimeDrawEngine.addNodes(imageNode); } public addVideo( diff --git a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts index df4e3446a7..d0ce985eeb 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts @@ -24,8 +24,6 @@ interface ShapeNodeConstructor { export class ShapeNode extends BaseNode { public kNode: Konva.Group; - private shapeType: ShapeType; - private shape: Konva.Shape; constructor({ canvasPosition, @@ -48,8 +46,8 @@ export class ShapeNode extends BaseNode { case ShapeType.CIRCLE: shape = new Konva.Circle({ radius: 50, - fill: color, ...transform, + fill: color, strokeScaleEnabled: false, draggable: true, }); @@ -81,12 +79,6 @@ export class ShapeNode extends BaseNode { throw new Error("Invalid shape type"); } - // Create group to satisfy BaseNode type requirements - // const group = new Konva.Group({ - // ...transform, - // draggable: true, - // }); - super({ selectionManagerRef: selectionManagerRef, mediaLayerRef: mediaLayerRef, diff --git a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts index 2c3f61521a..44b9382b77 100644 --- a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts +++ b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts @@ -28,7 +28,7 @@ export class RealTimeDrawEngine { // private frames: ImageBitmap[]; // capturing composite within window - private bgLayerRef: Konva.Layer; + private mediaLayerRef: Konva.Layer; private height: number; @@ -53,13 +53,11 @@ export class RealTimeDrawEngine { constructor({ width, height, - bgLayerRef, mediaLayerRef, offScreenCanvas, }: { width: number; height: number; - bgLayerRef: Konva.Layer; mediaLayerRef: Konva.Layer; offScreenCanvas: OffscreenCanvas; }) { @@ -82,7 +80,6 @@ export class RealTimeDrawEngine { this.offScreenCanvas.width = this.width; this.offScreenCanvas.height = this.height; - this.bgLayerRef = bgLayerRef; // this is the whole canvas this.mediaLayerRef = mediaLayerRef; @@ -118,16 +115,14 @@ export class RealTimeDrawEngine { stroke: "black", strokeWidth: 1, draggable: false, - fill: "red", + fill: "blue", }); - this.bgLayerRef.add(this.captureCanvas); - this.bgLayerRef.add(this.previewCanvas); + this.mediaLayerRef.add(this.captureCanvas); + this.mediaLayerRef.add(this.previewCanvas); // send back - this.captureCanvas.setZIndex(1); - this.previewCanvas.setZIndex(0); - - //this.debug(); + this.captureCanvas.setZIndex(0); + this.previewCanvas.setZIndex(1); } async updateCaptureCanvas( @@ -276,7 +271,9 @@ export class RealTimeDrawEngine { public addNodes(node: MediaNode) { if (node instanceof ImageNode || node instanceof TextNode || node instanceof ShapeNode) { + console.debug("Adding node:", node); this.imageNodes.push(node); + console.log(this.imageNodes) node.kNode.on("dragend", this.handleNodeDragEnd); } } @@ -447,7 +444,7 @@ export class RealTimeDrawEngine { } public async render() { - // only pick nodes that intersect with the canvas on screen bounds to freeze. + // only pick nodes that intersect wi th the canvas on screen bounds to freeze. this.mediaLayerRef.draw(); console.log( @@ -467,7 +464,7 @@ export class RealTimeDrawEngine { })) as ImageBitmap; // Test code - if (false) { + if (true) { this.outputBitmap = bitmap; this.previewCanvas.image(bitmap); this.isProcessing = false; From 1e38fbad5cf7d4d833932d0c5c4953cc7eff1e02 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Wed, 19 Feb 2025 20:33:01 -0500 Subject: [PATCH 4/4] Turns out you need to convert these rasters into images. --- frontend/realtime/src/KonvaApp/Engine.ts | 7 +- .../realtime/src/KonvaApp/Nodes/ShapeNode.ts | 78 ++++++++++++++++++- .../RenderingPrimitives/RealTimeDrawEngine.ts | 9 ++- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/frontend/realtime/src/KonvaApp/Engine.ts b/frontend/realtime/src/KonvaApp/Engine.ts index fac6ddd5f5..f0d1fd182c 100644 --- a/frontend/realtime/src/KonvaApp/Engine.ts +++ b/frontend/realtime/src/KonvaApp/Engine.ts @@ -180,11 +180,10 @@ export class Engine { this.setupEventSystem(); this.setAppMode(AppModes.SELECT); - - console.log("HELLO") + this.addShape(ShapeType.Circle,100); - // this.addShape(ShapeType.Square,100); - // this.addShape(ShapeType.Triangle,100); + this.addShape(ShapeType.Square,100); + this.addShape(ShapeType.Triangle,100); } diff --git a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts index d0ce985eeb..6e0b7520c5 100644 --- a/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts +++ b/frontend/realtime/src/KonvaApp/Nodes/ShapeNode.ts @@ -44,17 +44,41 @@ export class ShapeNode extends BaseNode { let shape = null; switch (shapeType) { case ShapeType.CIRCLE: - shape = new Konva.Circle({ + const circleShape = new Konva.Circle({ radius: 50, ...transform, fill: color, strokeScaleEnabled: false, draggable: true, }); + + // Convert circle to image + const dataURL = circleShape.toDataURL(); + const imageObj = new Image(); + imageObj.src = dataURL; + + shape = new Konva.Image({ + image: imageObj, + x: circleShape.x(), + y: circleShape.y(), + width: circleShape.width(), + height: circleShape.height(), + ...transform, + fill: 'transparent', + strokeScaleEnabled: false, + draggable: true, + }); + + imageObj.onload = () => { + this.mediaLayerRef.draw(); + }; + + // Clean up the temporary circle + circleShape.destroy(); break; case ShapeType.SQUARE: - shape = new Konva.Rect({ + const squareShape = new Konva.Rect({ width: 100, height: 100, ...transform, @@ -62,10 +86,34 @@ export class ShapeNode extends BaseNode { strokeScaleEnabled: false, draggable: true, }); + + // Convert square to image + const squareDataURL = squareShape.toDataURL(); + const squareImageObj = new Image(); + squareImageObj.src = squareDataURL; + + shape = new Konva.Image({ + image: squareImageObj, + x: squareShape.x(), + y: squareShape.y(), + width: squareShape.width(), + height: squareShape.height(), + ...transform, + fill: 'transparent', + strokeScaleEnabled: false, + draggable: true, + }); + + squareImageObj.onload = () => { + this.mediaLayerRef.draw(); + }; + + // Clean up the temporary square + squareShape.destroy(); break; case ShapeType.TRIANGLE: - shape = new Konva.RegularPolygon({ + const triangleShape = new Konva.RegularPolygon({ sides: 3, radius: 50, ...transform, @@ -73,6 +121,30 @@ export class ShapeNode extends BaseNode { strokeScaleEnabled: false, draggable: true, }); + + // Convert triangle to image + const triangleDataURL = triangleShape.toDataURL(); + const triangleImageObj = new Image(); + triangleImageObj.src = triangleDataURL; + + shape = new Konva.Image({ + image: triangleImageObj, + x: triangleShape.x(), + y: triangleShape.y(), + width: triangleShape.width(), + height: triangleShape.height(), + ...transform, + fill: 'transparent', + strokeScaleEnabled: false, + draggable: true, + }); + + triangleImageObj.onload = () => { + this.mediaLayerRef.draw(); + }; + + // Clean up the temporary triangle + triangleShape.destroy(); break; default: diff --git a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts index 44b9382b77..e5c69474d1 100644 --- a/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts +++ b/frontend/realtime/src/KonvaApp/RenderingPrimitives/RealTimeDrawEngine.ts @@ -99,7 +99,7 @@ export class RealTimeDrawEngine { y: this.positionY, width: this.width, height: this.height, - fill: "blue", + fill: "white", stroke: "black", strokeWidth: 1, draggable: false, @@ -115,7 +115,7 @@ export class RealTimeDrawEngine { stroke: "black", strokeWidth: 1, draggable: false, - fill: "blue", + fill: "white", }); this.mediaLayerRef.add(this.captureCanvas); @@ -309,6 +309,7 @@ export class RealTimeDrawEngine { try { const box = config.layerOfInterest.getClientRect(); const stage = config.layerOfInterest.getStage(); + const x = config.x !== undefined ? config.x : Math.floor(box.x); const y = config.y !== undefined ? config.y : Math.floor(box.y); @@ -446,7 +447,9 @@ export class RealTimeDrawEngine { public async render() { // only pick nodes that intersect wi th the canvas on screen bounds to freeze. this.mediaLayerRef.draw(); - + // Output all nodes in mediaLayerRef + const nodes = this.mediaLayerRef.getChildren(); + console.log("All nodes in mediaLayer:", nodes); console.log( `context: x:${this.positionX} y:${this.positionY} ${this.width} x ${this.height}`, );