diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index 50b3467ada..60b598ce5c 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -92,8 +92,8 @@ export class Engine { "", this.offScreenCanvas, this.videoLayer, - 1200, - 300, + 1550, + 450, "https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4", ); @@ -101,10 +101,12 @@ export class Engine { // Testing render engine this.renderEngine.addNodes(videoNode); + // await this.renderEngine.startProcessing(); + // Call this when test video nodes - //await this.renderEngine.startProcessing(); + // await this.renderEngine.startProcessing(); //videoNode.simulatedLoading(); // TODO support Text nodes diff --git a/src/KonvaApp/Nodes/VideoNode.ts b/src/KonvaApp/Nodes/VideoNode.ts index 0d664e205f..db973ac6a5 100644 --- a/src/KonvaApp/Nodes/VideoNode.ts +++ b/src/KonvaApp/Nodes/VideoNode.ts @@ -80,8 +80,8 @@ export class VideoNode extends NetworkedNodeContext { this.videoComponent = document.createElement("video"); + // Wrapping events this.frameDidFinishSeeking = new Promise(() => {}); - this.finishedLoadingOnStart = new Promise(() => {}); this.videoComponent.onloadedmetadata = (event: Event) => { @@ -94,20 +94,14 @@ export class VideoNode extends NetworkedNodeContext { this.duration = this.videoComponent.duration; }; - this.videoComponent.onseeked = (event: Event) => { - console.log("Seeked Finished"); - // reimplement using the function - // ensure that this doesn't race. - if (this.frameDidFinishSeeking) { - this.frameDidFinishSeeking.then(() => {}); - } - }; - - this.videoComponent.oncanplaythrough = (event: Event) => { - this.didFinishLoading = true; - // Might have to auto click? on first load this doesn't work in general how about after ? - this.videoComponent.play(); //sometimes race condition with the - }; + this.finishedLoadingOnStart = new Promise((resolve, reject) => { + this.videoComponent.oncanplaythrough = (event: Event) => { + this.didFinishLoading = true; + // Might have to auto click? on first load this doesn't work in general how about after ? + this.videoComponent.play(); //sometimes race condition with the + resolve(); + }; + }); // assign video to start process. this.videoComponent.src = this.videoURL; @@ -324,7 +318,6 @@ export class VideoNode extends NetworkedNodeContext { async seek(second: number) { // prevent interaction - //console.log(`${this.didFinishLoading}`); if (this.didFinishLoading === false) { console.log("Didn't finish loading so cannot seek"); return; @@ -335,7 +328,7 @@ export class VideoNode extends NetworkedNodeContext { console.log("Didn't setup Video Component?"); return; } - console.log("Setting"); + console.log(`Seeking to Position ${second}`); this.videoComponent.currentTime = second; @@ -347,7 +340,7 @@ export class VideoNode extends NetworkedNodeContext { resolve(); }; }); - // wait for this to finish + // await await this.frameDidFinishSeeking; } else { console.log("Video Not Seekable"); diff --git a/src/KonvaApp/RenderEngine.ts b/src/KonvaApp/RenderEngine.ts index fcb6516817..6b6d9284b0 100644 --- a/src/KonvaApp/RenderEngine.ts +++ b/src/KonvaApp/RenderEngine.ts @@ -1,7 +1,7 @@ -import { layer } from "@fortawesome/fontawesome-svg-core"; import { VideoNode } from "./Nodes/VideoNode"; import Konva from "konva"; -import { CanvasHTMLAttributes } from "react"; + +// https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio export class RenderEngine { private videoNodes: VideoNode[]; private offScreenCanvas: OffscreenCanvas; @@ -12,50 +12,59 @@ export class RenderEngine { // capturing composite within window private videoLayer: Konva.Layer; - private videoDrawCanvas: HTMLCanvasElement; - private videoDrawContext: CanvasRenderingContext2D | null; + + private height: number; + private width: number; + private positionX: number; + private positionY: number; + constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) { this.videoNodes = []; - this.offScreenCanvas = offScreenCanvas; - this.context = this.offScreenCanvas.getContext("2d"); - this.isProcessing = false; - this.frames = []; - // Rendering surface resolution + this.isProcessing = false; + + // TODO make this dynamic + + this.width = 720; + this.height = 1280; + this.positionX = window.innerWidth / 2 - 720 / 2; + this.positionY = window.innerHeight / 2 - 1080 / 2; + + this.offScreenCanvas = offScreenCanvas; + this.offScreenCanvas.width = this.width; + this.offScreenCanvas.height = this.height; + this.context = this.offScreenCanvas.getContext("2d"); + + this.frames = []; this.videoLayer = videoLayer; - var positionX = window.innerWidth / 2 - 720 / 2; - var positionY = window.innerHeight / 2 - 1080 / 2; - // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio - var width = 720; - var height = 1280; - const captureCanvas = new Konva.Rect({ - x: positionX, - y: positionY, - width: width, - height: height, + x: this.positionX, + y: this.positionY, + width: this.width, + height: this.height, fill: "white", stroke: "black", strokeWidth: 1, draggable: false, }); + + // DEBUG ONLY + // const rectangle = new Konva.Rect({ + // x: this.positionX, + // y: this.positionY, + // width: 100, + // height: 100, + // fill: "green", + // stroke: "black", + // strokeWidth: 1, + // draggable: false, + // }); + //this.videoLayer.add(rectangle); this.videoLayer.add(captureCanvas); // set furest back - captureCanvas.setZIndex(-10); - - // get the render position on the layer - this.videoDrawCanvas = this.renderPortionOfLayer( - this.videoLayer, - positionX, - positionY, - width, - height, - ); - - // sadly exists as a NOT AS a offscreen rendering context. ? maybe bitmaprenderer ? - this.videoDrawContext = this.videoDrawCanvas.getContext("2d"); + captureCanvas.setZIndex(0); } // This function uses a portion of the video layer to capture just the capture canvas. @@ -146,39 +155,61 @@ export class RenderEngine { // find longest video // then seek through each node 1 step. // stop ignore stepping if the duration is less. - private async render(numberOfFrames: number) { + private async render(largestNumberOfFrames: number) { if (!this.isProcessing) return; - for (let j = 0; j < numberOfFrames; j++) { - const videoNode = this.videoNodes[j]; + // Stop all nodes first + console.log(`LargestNumberOfFrames:${largestNumberOfFrames}`); + + for (let k = 0; k < this.videoNodes.length; k++) { + const videoNode = this.videoNodes[k]; + if (videoNode.didFinishLoading === false) { + throw Error("Videos Did Not Finish Loading Please Try Again."); + } videoNode.stop(); - const numberOfFrames = videoNode.getNumberFrames(); + } - for (let i = 0; i <= numberOfFrames; i++) { - const frameTime = this.calculateFrameTime(i, videoNode.fps); + for (let j = 0; j < largestNumberOfFrames; j++) { + for (let i = 0; i < this.videoNodes.length; i++) { + const currentVideoNode = this.videoNodes[i]; + const frameTime = this.calculateFrameTime(j, currentVideoNode.fps); - console.log(i); - console.log(frameTime); - console.log(videoNode.duration); + if (frameTime < currentVideoNode.duration) { + console.log(`CurrentFrame:${j}`); + console.log(`frameTime:${frameTime}`); + console.log(`duration:${currentVideoNode.duration}`); - if (frameTime < videoNode.duration) { - await videoNode.seek(frameTime); - this.offScreenCanvas.width = videoNode.node.getSize().width; - this.offScreenCanvas.height = videoNode.node.getSize().height; + await currentVideoNode.seek(frameTime); + this.videoLayer.draw(); - if (this.videoDrawContext) { - this.videoDrawContext.drawImage( - videoNode.videoComponent, + // SCOPES the capture for the context + // Correct size for the mobile canvas. + this.offScreenCanvas.width = this.width; + this.offScreenCanvas.height = this.height; + + if (this.context) { + // This crops it starting at position X / Y where the mobile canvas is + // Then picks the height and width range + // then we draw it at 0,0,width and height of the canvas + this.context.drawImage( + this.videoLayer.canvas._canvas, + this.positionX, + this.positionY, + this.width, + this.height, 0, 0, - this.offScreenCanvas.width, - this.offScreenCanvas.height, + this.width, + this.height, ); - // TODO bridge this. - //this.frames.push(this.videoDrawCanvas.toBlob()); - } - } // end of if - } // end of for. + const blob = await this.offScreenCanvas.convertToBlob({ + quality: 1.0, + type: "image/jpeg", + }); + this.blobToFile(blob); + } // end of if context + } // End frame time + } // end of for each frame } }