diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index b40b47125f..b6a5b9fd2f 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -22,7 +22,7 @@ import { TranslateCommand } from "./UndoRedo/TranslateCommand"; import { FileUtilities } from "./FileUtilities/FileUtilities"; -import { AppModes } from "./type"; +import { AppModes, VideoResolutions } from "./constants"; import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum"; export interface RenderingOptions { @@ -91,11 +91,14 @@ export class Engine { this.offScreenCanvas = new OffscreenCanvas(0, 0); - this.renderEngine = new RenderEngine( - this.videoLayer, - this.offScreenCanvas, - this.onRenderingSystemReceived.bind(this), - ); + this.renderEngine = new RenderEngine({ + width: VideoResolutions.VERTICAL_720.width, + height: VideoResolutions.VERTICAL_720.height, + videoLayer: this.videoLayer, + offScreenCanvas: this.offScreenCanvas, + onRenderingSystemMessageRecieved: + this.onRenderingSystemReceived.bind(this), + }); this.setupEventSystem(); } @@ -131,9 +134,9 @@ export class Engine { const media_url = `${media_api_base_url}vocodes-public${response.data.videoUrl}`; const videoNode = new VideoNode({ - videoLayer: this.videoLayer, - x: this.renderEngine.captureCanvas.position().x, - y: this.renderEngine.captureCanvas.position().y, + mediaLayer: this.videoLayer, + position: this.renderEngine.captureCanvas.position(), + canvasSize: this.renderEngine.captureCanvas.size(), videoURL: media_url, selectionManagerRef: this.selectionManager, nodeTransformerRef: this.nodeTransformer, @@ -281,27 +284,27 @@ export class Engine { ); const imageNode = new ImageNode({ mediaLayer: this.videoLayer, - x: this.renderEngine.captureCanvas.position().x, - y: this.renderEngine.captureCanvas.position().y, + position: this.renderEngine.captureCanvas.position(), + canvasSize: this.renderEngine.captureCanvas.size(), imageFile: imageFile, selectionManagerRef: this.selectionManager, - nodeTransformer: this.nodeTransformer, + nodeTransformerRef: this.nodeTransformer, }); const imageNode2 = new ImageNode({ mediaLayer: this.videoLayer, - x: this.renderEngine.captureCanvas.position().x, - y: this.renderEngine.captureCanvas.position().y, + position: this.renderEngine.captureCanvas.position(), + canvasSize: this.renderEngine.captureCanvas.size(), imageFile: imageFile, selectionManagerRef: this.selectionManager, - nodeTransformer: this.nodeTransformer, + nodeTransformerRef: this.nodeTransformer, }); // Adding nodes here const videoNode = new VideoNode({ - videoLayer: this.videoLayer, - x: this.renderEngine.captureCanvas.position().x, - y: this.renderEngine.captureCanvas.position().y, + mediaLayer: this.videoLayer, + position: this.renderEngine.captureCanvas.position(), + canvasSize: this.renderEngine.captureCanvas.size(), videoURL: "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", selectionManagerRef: this.selectionManager, @@ -353,11 +356,11 @@ export class Engine { public addImage(imageFile: File) { const imageNode = new ImageNode({ mediaLayer: this.videoLayer, - x: this.renderEngine.captureCanvas.position().x, - y: this.renderEngine.captureCanvas.position().y, + position: this.renderEngine.captureCanvas.position(), + canvasSize: this.renderEngine.captureCanvas.size(), imageFile: imageFile, selectionManagerRef: this.selectionManager, - nodeTransformer: this.nodeTransformer, + nodeTransformerRef: this.nodeTransformer, }); this.renderEngine.addNodes(imageNode); this.selectionManager.saveNode(imageNode); @@ -366,9 +369,9 @@ export class Engine { public addVideo(url: string) { // Adding nodes here const videoNode = new VideoNode({ - videoLayer: this.videoLayer, - x: this.renderEngine.captureCanvas.position().x, - y: this.renderEngine.captureCanvas.position().y, + mediaLayer: this.videoLayer, + position: this.renderEngine.captureCanvas.position(), + canvasSize: this.renderEngine.captureCanvas.size(), videoURL: url, selectionManagerRef: this.selectionManager, nodeTransformerRef: this.nodeTransformer, diff --git a/src/KonvaApp/NodeTransformer.ts b/src/KonvaApp/NodeTransformer.ts index c74a1c1946..a9d54c161a 100644 --- a/src/KonvaApp/NodeTransformer.ts +++ b/src/KonvaApp/NodeTransformer.ts @@ -1,6 +1,6 @@ import Konva from "konva"; import { Node, NodeConfig } from "konva/lib/Node"; -import { MediaNode } from "./type"; +import { MediaNode } from "./types"; export class NodeTransformer { private kTransformer: Konva.Transformer; diff --git a/src/KonvaApp/Nodes/ImageNode.ts b/src/KonvaApp/Nodes/ImageNode.ts index 7481a83734..5aba4a84c6 100644 --- a/src/KonvaApp/Nodes/ImageNode.ts +++ b/src/KonvaApp/Nodes/ImageNode.ts @@ -3,14 +3,15 @@ import { Layer } from "konva/lib/Layer"; import { NetworkedNodeContext } from "./NetworkedNodeContext"; import { SelectionManager } from "../SelectionManager"; import { NodeTransformer } from "../NodeTransformer"; +import { Position, Size } from "../types"; interface ImageNodeContructor { mediaLayer: Layer; - x: number; - y: number; + position: Position; + canvasSize: Size; imageFile: File; selectionManagerRef: SelectionManager; - nodeTransformer: NodeTransformer; + nodeTransformerRef: NodeTransformer; } export class ImageNode extends NetworkedNodeContext { @@ -20,14 +21,14 @@ export class ImageNode extends NetworkedNodeContext { constructor({ mediaLayer, - x, - y, + position, + canvasSize, imageFile, selectionManagerRef, - nodeTransformer, + nodeTransformerRef, }: ImageNodeContructor) { super({ - nodeTransfomerRef: nodeTransformer, + nodeTransfomerRef: nodeTransformerRef, selectionManagerRef: selectionManagerRef, mediaLayer: mediaLayer, }); @@ -40,8 +41,8 @@ export class ImageNode extends NetworkedNodeContext { this.imageObject.crossOrigin = "anonymous"; this.kNode = new Konva.Image({ - x: x, - y: y, + x: position.x, + y: position.y, image: undefined, // to do replace with placeholder width: 100, height: 100, @@ -56,9 +57,16 @@ export class ImageNode extends NetworkedNodeContext { return; } + const renderSize = this.calculateRenderSizeOnLoad({ + componentSize: { + width: imageComponent.width, + height: imageComponent.height, + }, + maxSize: canvasSize, + }); + this.kNode.image(imageComponent); - this.kNode.width(imageComponent.width); - this.kNode.height(imageComponent.height); + this.kNode.setSize(renderSize); this.listenToBaseKNode(); this.kNode.fill(null); diff --git a/src/KonvaApp/Nodes/NetworkedNodeContext.ts b/src/KonvaApp/Nodes/NetworkedNodeContext.ts index 7108b75de8..2fcf65102e 100644 --- a/src/KonvaApp/Nodes/NetworkedNodeContext.ts +++ b/src/KonvaApp/Nodes/NetworkedNodeContext.ts @@ -7,6 +7,7 @@ import { Node, NodeConfig } from "konva/lib/Node"; import { uiAccess } from "~/signals"; import { SelectionManager } from "../SelectionManager"; import { NodeTransformer } from "../NodeTransformer"; +import { Size } from "../types"; const toolbarNode = uiAccess.toolbarNode; const loadingBar = uiAccess.loadingBar; @@ -48,6 +49,29 @@ export class NetworkedNodeContext { this.kNode = undefined; this.didFinishLoading = false; } + protected calculateRenderSizeOnLoad({ + componentSize, + maxSize, + }: { + componentSize: Size; + maxSize: Size; + }) { + const renderSize = { + width: componentSize.width, + height: componentSize.height, + }; + if (renderSize.width > maxSize.width) { + const scaleDown = maxSize.width / renderSize.width; + renderSize.width = renderSize.width * scaleDown; + renderSize.height = renderSize.height * scaleDown; + } + if (renderSize.height > maxSize.height) { + const scaleDownAgain = maxSize.height / renderSize.height; + renderSize.width = renderSize.width * scaleDownAgain; + renderSize.height = renderSize.height * scaleDownAgain; + } + return renderSize; + } public delete() { // Do any other clean up and delete the konva node. if (this.kNode) { diff --git a/src/KonvaApp/Nodes/VideoNode.ts b/src/KonvaApp/Nodes/VideoNode.ts index c6d3ed883f..5c38c39378 100644 --- a/src/KonvaApp/Nodes/VideoNode.ts +++ b/src/KonvaApp/Nodes/VideoNode.ts @@ -4,10 +4,20 @@ import { NetworkedNodeContext } from "./NetworkedNodeContext"; import { uiAccess } from "~/signals"; import { SelectionManager } from "../SelectionManager"; import { NodeTransformer } from "../NodeTransformer"; +import { Position, Size } from "../types"; -const toolbarNode = uiAccess.toolbarNode; +// const toolbarNode = uiAccess.toolbarNode; const loadingBar = uiAccess.loadingBar; +interface VideoNodeContructor { + mediaLayer: Layer; + position: Position; + canvasSize: Size; + videoURL: string; + selectionManagerRef: SelectionManager; + nodeTransformerRef: NodeTransformer; +} + export class VideoNode extends NetworkedNodeContext { public videoURL: string; public videoComponent: HTMLVideoElement; @@ -40,24 +50,17 @@ export class VideoNode extends NetworkedNodeContext { private finishedLoadingOnStart: Promise; constructor({ - videoLayer, - x, - y, + mediaLayer, + position, + canvasSize, videoURL, selectionManagerRef, nodeTransformerRef, - }: { - videoLayer: Layer; - x: number; - y: number; - videoURL: string; - selectionManagerRef: SelectionManager; - nodeTransformerRef: NodeTransformer; - }) { + }: VideoNodeContructor) { super({ nodeTransfomerRef: nodeTransformerRef, selectionManagerRef: selectionManagerRef, - mediaLayer: videoLayer, + mediaLayer: mediaLayer, }); // state manage the node @@ -77,8 +80,8 @@ export class VideoNode extends NetworkedNodeContext { this.kNode = new Konva.Image({ image: undefined, - x: x, - y: y, + x: position.x, + y: position.y, width: 200, // to do fix this with placeholder height: 200, draggable: true, @@ -97,9 +100,17 @@ export class VideoNode extends NetworkedNodeContext { console.log("KNode Not Initialized Video Component"); return; } - this.kNode.height(this.videoComponent.videoHeight); - this.kNode.width(this.videoComponent.videoWidth); + + const renderSize = this.calculateRenderSizeOnLoad({ + componentSize: { + width: this.videoComponent.videoWidth, + height: this.videoComponent.videoHeight, + }, + maxSize: canvasSize, + }); + this.kNode.image(this.videoComponent); + this.kNode.setSize(renderSize); this.videoComponent.currentTime = 0; // ensure it shows up on screen // it might have length here which we will need to trim down to 7 seconds. diff --git a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts index 6b51528a83..e5c2146d38 100644 --- a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts +++ b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts @@ -60,6 +60,71 @@ export class RenderEngine { public videoLoadingCanvas: VideoNode | undefined; + constructor({ + width, + height, + videoLayer, + offScreenCanvas, + onRenderingSystemMessageRecieved, + }: { + width: number; + height: number; + videoLayer: Konva.Layer; + offScreenCanvas: OffscreenCanvas; + onRenderingSystemMessageRecieved: ( + response: SharedWorkerResponse< + DiffusionSharedWorkerResponseData, + DiffusionSharedWorkerProgressData + >, + ) => void; + }) { + this.videoLoadingCanvas = undefined; + this.videoNodes = []; + this.imageNodes = []; + + this.isProcessing = false; + this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved; + // TODO: Make this dynamic and update this on change of canvas. + + this.width = width; + this.height = height; + this.positionX = window.innerWidth / 2 - this.width / 2; + this.positionY = window.innerHeight / 2 - this.height / 2; + + this.offScreenCanvas = offScreenCanvas; + this.offScreenCanvas.width = this.width; + this.offScreenCanvas.height = this.height; + this.context = this.offScreenCanvas.getContext("2d"); + + this.frames = []; + + this.videoLayer = videoLayer; + + this.port = undefined; + this.captureCanvas = new Konva.Rect({ + x: this.positionX, + y: this.positionY, + width: this.width, + height: this.height, + fill: "white", + stroke: "black", + strokeWidth: 1, + draggable: false, + }); + this.captureCanvas.addName("CaptureCanvas"); + + this.upperMaxFrames = 7 * 24; + + this.videoLayer.add(this.captureCanvas); + // send back + this.captureCanvas.setZIndex(0); + + this.canUseSharedWorker = false; + this.setupSharedWorker(); + + //this.debug(); + } + private onRenderingSystemMessageRecieved: ( response: SharedWorkerResponse< DiffusionSharedWorkerResponseData, @@ -146,62 +211,6 @@ export class RenderEngine { }); this.videoLayer.batchDraw(); } - constructor( - videoLayer: Konva.Layer, - offScreenCanvas: OffscreenCanvas, - onRenderingSystemMessageRecieved: ( - response: SharedWorkerResponse< - DiffusionSharedWorkerResponseData, - DiffusionSharedWorkerProgressData - >, - ) => void, - ) { - this.videoLoadingCanvas = undefined; - this.videoNodes = []; - this.imageNodes = []; - - this.isProcessing = false; - this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved; - // TODO: Make this dynamic and update this on change of canvas. - - this.width = 720; - this.height = 1280; - this.positionX = window.innerWidth / 2 - this.width / 2; - this.positionY = window.innerHeight / 2 - this.height / 2; - - this.offScreenCanvas = offScreenCanvas; - this.offScreenCanvas.width = this.width; - this.offScreenCanvas.height = this.height; - this.context = this.offScreenCanvas.getContext("2d"); - - this.frames = []; - - this.videoLayer = videoLayer; - - this.port = undefined; - this.captureCanvas = new Konva.Rect({ - x: this.positionX, - y: this.positionY, - width: this.width, - height: this.height, - fill: "white", - stroke: "black", - strokeWidth: 1, - draggable: false, - }); - this.captureCanvas.addName("CaptureCanvas"); - - this.upperMaxFrames = 7 * 24; - - this.videoLayer.add(this.captureCanvas); - // send back - this.captureCanvas.setZIndex(0); - - this.canUseSharedWorker = false; - this.setupSharedWorker(); - - //this.debug(); - } debug() { // DEBUG ONLY diff --git a/src/KonvaApp/SelectionManager.ts b/src/KonvaApp/SelectionManager.ts index 2cb76f0394..2ea2e43a39 100644 --- a/src/KonvaApp/SelectionManager.ts +++ b/src/KonvaApp/SelectionManager.ts @@ -1,7 +1,7 @@ import Konva from "konva"; import { Node, NodeConfig } from "konva/lib/Node"; -import { MediaNode } from "./type"; +import { MediaNode } from "./types"; import { uiAccess } from "~/signals"; export class SelectionManager { diff --git a/src/KonvaApp/constants.ts b/src/KonvaApp/constants.ts new file mode 100644 index 0000000000..4ce31441cc --- /dev/null +++ b/src/KonvaApp/constants.ts @@ -0,0 +1,7 @@ +export enum AppModes { + SELECT = "SELECT", +} + +export const VideoResolutions = { + VERTICAL_720: { width: 720, height: 1280 }, +}; diff --git a/src/KonvaApp/type.ts b/src/KonvaApp/types.ts similarity index 67% rename from src/KonvaApp/type.ts rename to src/KonvaApp/types.ts index 3ae281ff0e..b04a8bddeb 100644 --- a/src/KonvaApp/type.ts +++ b/src/KonvaApp/types.ts @@ -2,7 +2,11 @@ import { VideoNode } from "./Nodes/VideoNode"; import { ImageNode } from "./Nodes/ImageNode"; import { NetworkedNodeContext } from "./Nodes/NetworkedNodeContext"; export type MediaNode = NetworkedNodeContext | VideoNode | ImageNode; - -export enum AppModes { - SELECT = "SELECT", +export interface Size { + width: number; + height: number; +} +export interface Position { + x: number; + y: number; }