From aafa6fdb605a44a9dc43e901658da1f199c66c55 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:10:25 -0400 Subject: [PATCH] Have to test the new canvas bounds render technique. --- src/KonvaApp/Engine.ts | 139 ++++------------------------------- src/KonvaApp/RenderEngine.ts | 114 +++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 129 deletions(-) diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index 4ffdc1c088..ccf24fe688 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -3,7 +3,7 @@ import { VideoNode } from "./Nodes/VideoNode"; import { uiAccess } from "~/signals"; import { uiEvents } from "~/signals"; import { RenderEngine } from "./RenderEngine"; -import { layer } from "@fortawesome/fontawesome-svg-core"; + export class Engine { private canvasReference: HTMLDivElement; private stage: Konva.Stage; @@ -19,6 +19,9 @@ export class Engine { width: window.innerWidth, height: window.innerHeight, }); + const videoLayer = new Konva.Layer(); + this.videoLayer = videoLayer; + this.stage.add(videoLayer); // Listen to changes in container size const resizeObserver = new ResizeObserver(() => { @@ -28,12 +31,10 @@ export class Engine { resizeObserver.observe(this.canvasReference); this.applyChanges(); - const videoLayer = new Konva.Layer(); - this.videoLayer = videoLayer; - this.stage.add(videoLayer); + // core layer for all the work done. this.offScreenCanvas = new OffscreenCanvas(0, 0); - const context = this.offScreenCanvas.getContext("2d"); + this.renderEngine = new RenderEngine(this.videoLayer, this.offScreenCanvas); uiEvents.onGetStagedImage((image) => { this.addImage(image); @@ -54,7 +55,7 @@ export class Engine { } public initializeStage(sceneToken: string) { - // load canvas that was originaly saved + // load canvas that was originaly saved TODO Save manager for resharing. uiAccess.imageToolbar.hide(); uiAccess.loadingBar.hide(); @@ -92,101 +93,21 @@ export class Engine { "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", ); - const node2 = new VideoNode( - "", - this.offScreenCanvas, - this.videoLayer, - 1600, - 700, - "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", - ); + // CODE TO TEST RENDER ENGINE + // Testing render engine + this.renderEngine.addNodes(videoNode); - var positionX = window.innerWidth / 2 - 720 / 2; - var positionY = window.innerHeight / 2 - 1080 / 2; + await this.renderEngine.render(); - var width = 720; - var height = 1080; - - const activeFrame = new Konva.Rect({ - x: positionX, - y: positionY, - width: width, - height: height, - fill: "white", - stroke: "black", - strokeWidth: 1, - draggable: false, - }); - - const activeFrame2 = new Konva.Rect({ - x: positionX, - y: positionY, - width: 100, - height: 100, - fill: "green", - stroke: "black", - strokeWidth: 1, - draggable: false, - }); - - this.videoLayer.add(activeFrame); - this.videoLayer.add(activeFrame2); - activeFrame2.moveToTop(); - activeFrame.moveToBottom(); - - await this.sleep(5000); - - for (let i = 0; i < 10; i++) { - const canvas = await this.renderPortionOfLayer( - this.videoLayer, - positionX, - positionY, - width, - height, - ); - - canvas.toBlob((blob) => { - if (blob) { - // Do something with the blob, e.g., upload it or save it - console.log("Blob created:", blob); - // Example: Create a download link for the blob - const link = document.createElement("a"); - link.href = URL.createObjectURL(blob); - link.download = "canvas-output.png"; - link.click(); - } else { - console.error("Failed to create blob"); - } - }, "image/png"); - - await this.sleep(2000); - } - //this.renderEngine.addNodes(videoNode); - - // Call this when + // Call this when test video nodes //await this.renderEngine.startProcessing(); //videoNode.simulatedLoading(); + // TODO support Text nodes this.videoLayer.add(textNode); } - async renderPortionOfLayer( - layer: Konva.Layer, - x: number, - y: number, - width: number, - height: number, - ) { - const canvas = layer.toCanvas({ - x: x, - y: y, - width: width, - height: height, - }); - return canvas; - } - public addImage(imageFile: File) { // main API: const imageObj = new Image(); @@ -210,37 +131,3 @@ export class Engine { console.log("addVideo", videoFile); } } - -// do nothing, animation just need to update the layer -// try { -// if (!context) { -// console.log("Context is dead."); -// return; -// } -// offScreenCanvas.width = videoNode.getWidth(); -// offScreenCanvas.height = videoNode.getHeight(); -// context.drawImage( -// video, -// 0, -// 0, -// videoNode.getWidth(), -// videoNode.getHeight(), -// ); -// console.log("draw"); -// const blob = await offScreenCanvas.convertToBlob({ -// quality: 1.0, -// type: "image/jpeg", -// }); -// console.log("blob"); -// const link = document.createElement("a"); -// link.href = URL.createObjectURL(blob); -// link.download = "canvas-output.jpg"; -// // Trigger the download -// link.click(); -// // Clean up the URL object -// URL.revokeObjectURL(link.href); -// console.log("hello"); -// } catch (error) { -// console.log(error); -// } -// anim.stop(); diff --git a/src/KonvaApp/RenderEngine.ts b/src/KonvaApp/RenderEngine.ts index b23e7747ec..6296e02419 100644 --- a/src/KonvaApp/RenderEngine.ts +++ b/src/KonvaApp/RenderEngine.ts @@ -1,4 +1,7 @@ +import { layer } from "@fortawesome/fontawesome-svg-core"; import { VideoNode } from "./Nodes/VideoNode"; +import Konva from "konva"; +import { CanvasHTMLAttributes } from "react"; export class RenderEngine { private videoNodes: VideoNode[]; private offScreenCanvas: OffscreenCanvas; @@ -6,12 +9,71 @@ export class RenderEngine { private isProcessing: boolean; private frames: ImageBitmap[]; - constructor(offScreenCanvas: OffscreenCanvas) { + + // capturing composite within window + private videoLayer: Konva.Layer; + private videoDrawCanvas: HTMLCanvasElement; + private videoDrawContext: CanvasRenderingContext2D | null; + constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) { this.videoNodes = []; this.offScreenCanvas = offScreenCanvas; this.context = this.offScreenCanvas.getContext("2d"); this.isProcessing = false; this.frames = []; + + // Rendering surface resolution + + this.videoLayer = videoLayer; + + var positionX = window.innerWidth / 2 - 720 / 2; + var positionY = window.innerHeight / 2 - 1080 / 2; + // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio + var width = 720; + var height = 1280; + + const captureCanvas = new Konva.Rect({ + x: positionX, + y: positionY, + width: width, + height: height, + fill: "white", + stroke: "black", + strokeWidth: 1, + draggable: false, + }); + this.videoLayer.add(captureCanvas); + // set furest back + captureCanvas.setZIndex(-10); + + // get the render position on the layer + this.videoDrawCanvas = this.renderPortionOfLayer( + this.videoLayer, + positionX, + positionY, + width, + height, + ); + + // sadly exists as a NOT AS a offscreen rendering context. ? maybe bitmaprenderer ? + this.videoDrawContext = this.videoDrawCanvas.getContext("2d"); + } + + // This function uses a portion of the video layer to capture just the capture canvas. + // capture everything after seeking each video node. + renderPortionOfLayer( + layer: Konva.Layer, + x: number, + y: number, + width: number, + height: number, + ): HTMLCanvasElement { + const canvas = layer.toCanvas({ + x: x, + y: y, + width: width, + height: height, + }); + return canvas; } private findLongestVideoLength(): number { @@ -62,9 +124,13 @@ export class RenderEngine { this.videoNodes.forEach((item: VideoNode) => {}); - // find the longest video + // find the longest video node; const numberOfFrames = this.findLongestVideoLength(); - await this.processFrame(); + + await this.render(numberOfFrames); + + // only to test video node + //await this.processFrame(); } public stopProcessing() { @@ -76,6 +142,48 @@ export class RenderEngine { return frameNumber / frameRate; } + // TODO render loop should be. + // find longest video + // then seek through each node 1 step. + // stop ignore stepping if the duration is less. + private async render(numberOfFrames: number) { + if (!this.isProcessing) return; + for (let j = 0; j < numberOfFrames; j++) { + const videoNode = this.videoNodes[j]; + + videoNode.stop(); + const numberOfFrames = videoNode.getNumberFrames(); + + for (let i = 0; i <= numberOfFrames; i++) { + const frameTime = this.calculateFrameTime(i, videoNode.fps); + + console.log(i); + console.log(frameTime); + console.log(videoNode.duration); + + if (frameTime < videoNode.duration) { + await videoNode.seek(frameTime); + this.offScreenCanvas.width = videoNode.node.getSize().width; + this.offScreenCanvas.height = videoNode.node.getSize().height; + + if (this.context) { + this.context.drawImage( + videoNode.videoComponent, + 0, + 0, + this.offScreenCanvas.width, + this.offScreenCanvas.height, + ); + + this.frames.push(this.offScreenCanvas.transferToImageBitmap()); + } + } // end of if + } // end of for. + } + } + + //DO NOT USE: This code is to test whether or not we can seek frames on each video nodes. + //To check the correctness of the seeking. private async processFrame() { if (!this.isProcessing) return;