Merge pull request #32 from storytold/feature/reize-node-onadd

Feature /resize node on-add

src/KonvaApp/Engine.ts

@@ -22,7 +22,7 @@ import { TranslateCommand } from "./UndoRedo/TranslateCommand";
 
 import { FileUtilities } from "./FileUtilities/FileUtilities";
 
-import { AppModes } from "./type";
+import { AppModes, VideoResolutions } from "./constants";
 import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
 
 export interface RenderingOptions {
@@ -91,11 +91,14 @@ export class Engine {
 
     this.offScreenCanvas = new OffscreenCanvas(0, 0);
 
-    this.renderEngine = new RenderEngine(
-      this.videoLayer,
-      this.offScreenCanvas,
-      this.onRenderingSystemReceived.bind(this),
-    );
+    this.renderEngine = new RenderEngine({
+      width: VideoResolutions.VERTICAL_720.width,
+      height: VideoResolutions.VERTICAL_720.height,
+      videoLayer: this.videoLayer,
+      offScreenCanvas: this.offScreenCanvas,
+      onRenderingSystemMessageRecieved:
+        this.onRenderingSystemReceived.bind(this),
+    });
 
     this.setupEventSystem();
   }
@@ -131,9 +134,9 @@ export class Engine {
       const media_url = `${media_api_base_url}vocodes-public${response.data.videoUrl}`;
 
       const videoNode = new VideoNode({
-        videoLayer: this.videoLayer,
-        x: this.renderEngine.captureCanvas.position().x,
-        y: this.renderEngine.captureCanvas.position().y,
+        mediaLayer: this.videoLayer,
+        position: this.renderEngine.captureCanvas.position(),
+        canvasSize: this.renderEngine.captureCanvas.size(),
         videoURL: media_url,
         selectionManagerRef: this.selectionManager,
         nodeTransformerRef: this.nodeTransformer,
@@ -281,27 +284,27 @@ export class Engine {
     );
     const imageNode = new ImageNode({
       mediaLayer: this.videoLayer,
-      x: this.renderEngine.captureCanvas.position().x,
-      y: this.renderEngine.captureCanvas.position().y,
+      position: this.renderEngine.captureCanvas.position(),
+      canvasSize: this.renderEngine.captureCanvas.size(),
       imageFile: imageFile,
       selectionManagerRef: this.selectionManager,
-      nodeTransformer: this.nodeTransformer,
+      nodeTransformerRef: this.nodeTransformer,
     });
 
     const imageNode2 = new ImageNode({
       mediaLayer: this.videoLayer,
-      x: this.renderEngine.captureCanvas.position().x,
-      y: this.renderEngine.captureCanvas.position().y,
+      position: this.renderEngine.captureCanvas.position(),
+      canvasSize: this.renderEngine.captureCanvas.size(),
       imageFile: imageFile,
       selectionManagerRef: this.selectionManager,
-      nodeTransformer: this.nodeTransformer,
+      nodeTransformerRef: this.nodeTransformer,
     });
 
     // Adding nodes here
     const videoNode = new VideoNode({
-      videoLayer: this.videoLayer,
-      x: this.renderEngine.captureCanvas.position().x,
-      y: this.renderEngine.captureCanvas.position().y,
+      mediaLayer: this.videoLayer,
+      position: this.renderEngine.captureCanvas.position(),
+      canvasSize: this.renderEngine.captureCanvas.size(),
       videoURL:
         "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4",
       selectionManagerRef: this.selectionManager,
@@ -353,11 +356,11 @@ export class Engine {
   public addImage(imageFile: File) {
     const imageNode = new ImageNode({
       mediaLayer: this.videoLayer,
-      x: this.renderEngine.captureCanvas.position().x,
-      y: this.renderEngine.captureCanvas.position().y,
+      position: this.renderEngine.captureCanvas.position(),
+      canvasSize: this.renderEngine.captureCanvas.size(),
       imageFile: imageFile,
       selectionManagerRef: this.selectionManager,
-      nodeTransformer: this.nodeTransformer,
+      nodeTransformerRef: this.nodeTransformer,
     });
     this.renderEngine.addNodes(imageNode);
     this.selectionManager.saveNode(imageNode);
@@ -366,9 +369,9 @@ export class Engine {
   public addVideo(url: string) {
     // Adding nodes here
     const videoNode = new VideoNode({
-      videoLayer: this.videoLayer,
-      x: this.renderEngine.captureCanvas.position().x,
-      y: this.renderEngine.captureCanvas.position().y,
+      mediaLayer: this.videoLayer,
+      position: this.renderEngine.captureCanvas.position(),
+      canvasSize: this.renderEngine.captureCanvas.size(),
       videoURL: url,
       selectionManagerRef: this.selectionManager,
       nodeTransformerRef: this.nodeTransformer,

src/KonvaApp/NodeTransformer.ts

@@ -1,6 +1,6 @@
 import Konva from "konva";
 import { Node, NodeConfig } from "konva/lib/Node";
-import { MediaNode } from "./type";
+import { MediaNode } from "./types";
 
 export class NodeTransformer {
   private kTransformer: Konva.Transformer;

src/KonvaApp/Nodes/ImageNode.ts

@@ -3,14 +3,15 @@ import { Layer } from "konva/lib/Layer";
 import { NetworkedNodeContext } from "./NetworkedNodeContext";
 import { SelectionManager } from "../SelectionManager";
 import { NodeTransformer } from "../NodeTransformer";
+import { Position, Size } from "../types";
 
 interface ImageNodeContructor {
   mediaLayer: Layer;
-  x: number;
-  y: number;
+  position: Position;
+  canvasSize: Size;
   imageFile: File;
   selectionManagerRef: SelectionManager;
-  nodeTransformer: NodeTransformer;
+  nodeTransformerRef: NodeTransformer;
 }
 
 export class ImageNode extends NetworkedNodeContext {
@@ -20,14 +21,14 @@ export class ImageNode extends NetworkedNodeContext {
 
   constructor({
     mediaLayer,
-    x,
-    y,
+    position,
+    canvasSize,
     imageFile,
     selectionManagerRef,
-    nodeTransformer,
+    nodeTransformerRef,
   }: ImageNodeContructor) {
     super({
-      nodeTransfomerRef: nodeTransformer,
+      nodeTransfomerRef: nodeTransformerRef,
       selectionManagerRef: selectionManagerRef,
       mediaLayer: mediaLayer,
     });
@@ -40,8 +41,8 @@ export class ImageNode extends NetworkedNodeContext {
     this.imageObject.crossOrigin = "anonymous";
 
     this.kNode = new Konva.Image({
-      x: x,
-      y: y,
+      x: position.x,
+      y: position.y,
       image: undefined, // to do replace with placeholder
       width: 100,
       height: 100,
@@ -56,9 +57,16 @@ export class ImageNode extends NetworkedNodeContext {
         return;
       }
 
+      const renderSize = this.calculateRenderSizeOnLoad({
+        componentSize: {
+          width: imageComponent.width,
+          height: imageComponent.height,
+        },
+        maxSize: canvasSize,
+      });
+
       this.kNode.image(imageComponent);
-      this.kNode.width(imageComponent.width);
-      this.kNode.height(imageComponent.height);
+      this.kNode.setSize(renderSize);
 
       this.listenToBaseKNode();
       this.kNode.fill(null);

src/KonvaApp/Nodes/NetworkedNodeContext.ts

@@ -7,6 +7,7 @@ import { Node, NodeConfig } from "konva/lib/Node";
 import { uiAccess } from "~/signals";
 import { SelectionManager } from "../SelectionManager";
 import { NodeTransformer } from "../NodeTransformer";
+import { Size } from "../types";
 
 const toolbarNode = uiAccess.toolbarNode;
 const loadingBar = uiAccess.loadingBar;
@@ -48,6 +49,29 @@ export class NetworkedNodeContext {
     this.kNode = undefined;
     this.didFinishLoading = false;
   }
+  protected calculateRenderSizeOnLoad({
+    componentSize,
+    maxSize,
+  }: {
+    componentSize: Size;
+    maxSize: Size;
+  }) {
+    const renderSize = {
+      width: componentSize.width,
+      height: componentSize.height,
+    };
+    if (renderSize.width > maxSize.width) {
+      const scaleDown = maxSize.width / renderSize.width;
+      renderSize.width = renderSize.width * scaleDown;
+      renderSize.height = renderSize.height * scaleDown;
+    }
+    if (renderSize.height > maxSize.height) {
+      const scaleDownAgain = maxSize.height / renderSize.height;
+      renderSize.width = renderSize.width * scaleDownAgain;
+      renderSize.height = renderSize.height * scaleDownAgain;
+    }
+    return renderSize;
+  }
   public delete() {
     // Do any other clean up and delete the konva node.
     if (this.kNode) {

src/KonvaApp/Nodes/VideoNode.ts

@@ -4,10 +4,20 @@ import { NetworkedNodeContext } from "./NetworkedNodeContext";
 import { uiAccess } from "~/signals";
 import { SelectionManager } from "../SelectionManager";
 import { NodeTransformer } from "../NodeTransformer";
+import { Position, Size } from "../types";
 
-const toolbarNode = uiAccess.toolbarNode;
+// const toolbarNode = uiAccess.toolbarNode;
 const loadingBar = uiAccess.loadingBar;
 
+interface VideoNodeContructor {
+  mediaLayer: Layer;
+  position: Position;
+  canvasSize: Size;
+  videoURL: string;
+  selectionManagerRef: SelectionManager;
+  nodeTransformerRef: NodeTransformer;
+}
+
 export class VideoNode extends NetworkedNodeContext {
   public videoURL: string;
   public videoComponent: HTMLVideoElement;
@@ -40,24 +50,17 @@ export class VideoNode extends NetworkedNodeContext {
   private finishedLoadingOnStart: Promise<void>;
 
   constructor({
-    videoLayer,
-    x,
-    y,
+    mediaLayer,
+    position,
+    canvasSize,
     videoURL,
     selectionManagerRef,
     nodeTransformerRef,
-  }: {
-    videoLayer: Layer;
-    x: number;
-    y: number;
-    videoURL: string;
-    selectionManagerRef: SelectionManager;
-    nodeTransformerRef: NodeTransformer;
-  }) {
+  }: VideoNodeContructor) {
     super({
       nodeTransfomerRef: nodeTransformerRef,
       selectionManagerRef: selectionManagerRef,
-      mediaLayer: videoLayer,
+      mediaLayer: mediaLayer,
     });
 
     // state manage the node
@@ -77,8 +80,8 @@ export class VideoNode extends NetworkedNodeContext {
 
     this.kNode = new Konva.Image({
       image: undefined,
-      x: x,
-      y: y,
+      x: position.x,
+      y: position.y,
       width: 200, // to do fix this with placeholder
       height: 200,
       draggable: true,
@@ -97,9 +100,17 @@ export class VideoNode extends NetworkedNodeContext {
         console.log("KNode Not Initialized Video Component");
         return;
       }
-      this.kNode.height(this.videoComponent.videoHeight);
-      this.kNode.width(this.videoComponent.videoWidth);
+
+      const renderSize = this.calculateRenderSizeOnLoad({
+        componentSize: {
+          width: this.videoComponent.videoWidth,
+          height: this.videoComponent.videoHeight,
+        },
+        maxSize: canvasSize,
+      });
+
       this.kNode.image(this.videoComponent);
+      this.kNode.setSize(renderSize);
 
       this.videoComponent.currentTime = 0; // ensure it shows up on screen
       // it might have length here which we will need to trim down to 7 seconds.

src/KonvaApp/RenderingPrimitives/RenderEngine.ts

@@ -60,6 +60,71 @@ export class RenderEngine {
 
   public videoLoadingCanvas: VideoNode | undefined;
 
+  constructor({
+    width,
+    height,
+    videoLayer,
+    offScreenCanvas,
+    onRenderingSystemMessageRecieved,
+  }: {
+    width: number;
+    height: number;
+    videoLayer: Konva.Layer;
+    offScreenCanvas: OffscreenCanvas;
+    onRenderingSystemMessageRecieved: (
+      response: SharedWorkerResponse<
+        DiffusionSharedWorkerResponseData,
+        DiffusionSharedWorkerProgressData
+      >,
+    ) => void;
+  }) {
+    this.videoLoadingCanvas = undefined;
+    this.videoNodes = [];
+    this.imageNodes = [];
+
+    this.isProcessing = false;
+    this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved;
+    // TODO: Make this dynamic and update this on change of canvas.
+
+    this.width = width;
+    this.height = height;
+    this.positionX = window.innerWidth / 2 - this.width / 2;
+    this.positionY = window.innerHeight / 2 - this.height / 2;
+
+    this.offScreenCanvas = offScreenCanvas;
+    this.offScreenCanvas.width = this.width;
+    this.offScreenCanvas.height = this.height;
+    this.context = this.offScreenCanvas.getContext("2d");
+
+    this.frames = [];
+
+    this.videoLayer = videoLayer;
+
+    this.port = undefined;
+    this.captureCanvas = new Konva.Rect({
+      x: this.positionX,
+      y: this.positionY,
+      width: this.width,
+      height: this.height,
+      fill: "white",
+      stroke: "black",
+      strokeWidth: 1,
+      draggable: false,
+    });
+    this.captureCanvas.addName("CaptureCanvas");
+
+    this.upperMaxFrames = 7 * 24;
+
+    this.videoLayer.add(this.captureCanvas);
+    // send back
+    this.captureCanvas.setZIndex(0);
+
+    this.canUseSharedWorker = false;
+    this.setupSharedWorker();
+
+    //this.debug();
+  }
+
   private onRenderingSystemMessageRecieved: (
     response: SharedWorkerResponse<
       DiffusionSharedWorkerResponseData,
@@ -146,62 +211,6 @@ export class RenderEngine {
     });
     this.videoLayer.batchDraw();
   }
-  constructor(
-    videoLayer: Konva.Layer,
-    offScreenCanvas: OffscreenCanvas,
-    onRenderingSystemMessageRecieved: (
-      response: SharedWorkerResponse<
-        DiffusionSharedWorkerResponseData,
-        DiffusionSharedWorkerProgressData
-      >,
-    ) => void,
-  ) {
-    this.videoLoadingCanvas = undefined;
-    this.videoNodes = [];
-    this.imageNodes = [];
-
-    this.isProcessing = false;
-    this.onRenderingSystemMessageRecieved = onRenderingSystemMessageRecieved;
-    // TODO: Make this dynamic and update this on change of canvas.
-
-    this.width = 720;
-    this.height = 1280;
-    this.positionX = window.innerWidth / 2 - this.width / 2;
-    this.positionY = window.innerHeight / 2 - this.height / 2;
-
-    this.offScreenCanvas = offScreenCanvas;
-    this.offScreenCanvas.width = this.width;
-    this.offScreenCanvas.height = this.height;
-    this.context = this.offScreenCanvas.getContext("2d");
-
-    this.frames = [];
-
-    this.videoLayer = videoLayer;
-
-    this.port = undefined;
-    this.captureCanvas = new Konva.Rect({
-      x: this.positionX,
-      y: this.positionY,
-      width: this.width,
-      height: this.height,
-      fill: "white",
-      stroke: "black",
-      strokeWidth: 1,
-      draggable: false,
-    });
-    this.captureCanvas.addName("CaptureCanvas");
-
-    this.upperMaxFrames = 7 * 24;
-
-    this.videoLayer.add(this.captureCanvas);
-    // send back
-    this.captureCanvas.setZIndex(0);
-
-    this.canUseSharedWorker = false;
-    this.setupSharedWorker();
-
-    //this.debug();
-  }
 
   debug() {
     // DEBUG ONLY

src/KonvaApp/SelectionManager.ts

@@ -1,7 +1,7 @@
 import Konva from "konva";
 import { Node, NodeConfig } from "konva/lib/Node";
 
-import { MediaNode } from "./type";
+import { MediaNode } from "./types";
 import { uiAccess } from "~/signals";
 
 export class SelectionManager {

src/KonvaApp/constants.ts

@@ -0,0 +1,7 @@
+export enum AppModes {
+  SELECT = "SELECT",
+}
+
+export const VideoResolutions = {
+  VERTICAL_720: { width: 720, height: 1280 },
+};

src/KonvaApp/types.ts

@@ -2,7 +2,11 @@ import { VideoNode } from "./Nodes/VideoNode";
 import { ImageNode } from "./Nodes/ImageNode";
 import { NetworkedNodeContext } from "./Nodes/NetworkedNodeContext";
 export type MediaNode = NetworkedNodeContext | VideoNode | ImageNode;
-
-export enum AppModes {
-  SELECT = "SELECT",
+export interface Size {
+  width: number;
+  height: number;
+}
+export interface Position {
+  x: number;
+  y: number;
 }