Have to test the new canvas bounds render technique.

This commit is contained in:
Michael Chung committed 2024-09-07 02:10:25 -04:00
1 parent 9424e272e2
commit aafa6fdb60
2 files changed
+124 -129

No files matched your search

+13 -126
View File
@@ -3,7 +3,7 @@ import { VideoNode } from "./Nodes/VideoNode";
import { uiAccess } from "~/signals";
import { uiEvents } from "~/signals";
import { RenderEngine } from "./RenderEngine";
import { layer } from "@fortawesome/fontawesome-svg-core";
export class Engine {
private canvasReference: HTMLDivElement;
private stage: Konva.Stage;
@@ -19,6 +19,9 @@ export class Engine {
width: window.innerWidth,
height: window.innerHeight,
});
const videoLayer = new Konva.Layer();
this.videoLayer = videoLayer;
this.stage.add(videoLayer);
// Listen to changes in container size
const resizeObserver = new ResizeObserver(() => {
@@ -28,12 +31,10 @@ export class Engine {
resizeObserver.observe(this.canvasReference);
this.applyChanges();
const videoLayer = new Konva.Layer();
this.videoLayer = videoLayer;
this.stage.add(videoLayer);
// core layer for all the work done.
this.offScreenCanvas = new OffscreenCanvas(0, 0);
const context = this.offScreenCanvas.getContext("2d");
this.renderEngine = new RenderEngine(this.videoLayer, this.offScreenCanvas);
uiEvents.onGetStagedImage((image) => {
this.addImage(image);
@@ -54,7 +55,7 @@ export class Engine {
}
public initializeStage(sceneToken: string) {
// load canvas that was originaly saved
// load canvas that was originaly saved TODO Save manager for resharing.
uiAccess.imageToolbar.hide();
uiAccess.loadingBar.hide();
@@ -92,101 +93,21 @@ export class Engine {
"https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4",
);
const node2 = new VideoNode(
"",
this.offScreenCanvas,
this.videoLayer,
1600,
700,
"https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4",
);
// CODE TO TEST RENDER ENGINE
// Testing render engine
this.renderEngine.addNodes(videoNode);
var positionX = window.innerWidth / 2 - 720 / 2;
var positionY = window.innerHeight / 2 - 1080 / 2;
await this.renderEngine.render();
var width = 720;
var height = 1080;
const activeFrame = new Konva.Rect({
x: positionX,
y: positionY,
width: width,
height: height,
fill: "white",
stroke: "black",
strokeWidth: 1,
draggable: false,
});
const activeFrame2 = new Konva.Rect({
x: positionX,
y: positionY,
width: 100,
height: 100,
fill: "green",
stroke: "black",
strokeWidth: 1,
draggable: false,
});
this.videoLayer.add(activeFrame);
this.videoLayer.add(activeFrame2);
activeFrame2.moveToTop();
activeFrame.moveToBottom();
await this.sleep(5000);
for (let i = 0; i < 10; i++) {
const canvas = await this.renderPortionOfLayer(
this.videoLayer,
positionX,
positionY,
width,
height,
);
canvas.toBlob((blob) => {
if (blob) {
// Do something with the blob, e.g., upload it or save it
console.log("Blob created:", blob);
// Example: Create a download link for the blob
const link = document.createElement("a");
link.href = URL.createObjectURL(blob);
link.download = "canvas-output.png";
link.click();
} else {
console.error("Failed to create blob");
}
}, "image/png");
await this.sleep(2000);
}
//this.renderEngine.addNodes(videoNode);
// Call this when
// Call this when test video nodes
//await this.renderEngine.startProcessing();
//videoNode.simulatedLoading();
// TODO support Text nodes
this.videoLayer.add(textNode);
}
async renderPortionOfLayer(
layer: Konva.Layer,
x: number,
y: number,
width: number,
height: number,
) {
const canvas = layer.toCanvas({
x: x,
y: y,
width: width,
height: height,
});
return canvas;
}
public addImage(imageFile: File) {
// main API:
const imageObj = new Image();
@@ -210,37 +131,3 @@ export class Engine {
console.log("addVideo", videoFile);
}
}
// do nothing, animation just need to update the layer
// try {
// if (!context) {
// console.log("Context is dead.");
// return;
// }
// offScreenCanvas.width = videoNode.getWidth();
// offScreenCanvas.height = videoNode.getHeight();
// context.drawImage(
// video,
// 0,
// 0,
// videoNode.getWidth(),
// videoNode.getHeight(),
// );
// console.log("draw");
// const blob = await offScreenCanvas.convertToBlob({
// quality: 1.0,
// type: "image/jpeg",
// });
// console.log("blob");
// const link = document.createElement("a");
// link.href = URL.createObjectURL(blob);
// link.download = "canvas-output.jpg";
// // Trigger the download
// link.click();
// // Clean up the URL object
// URL.revokeObjectURL(link.href);
// console.log("hello");
// } catch (error) {
// console.log(error);
// }
// anim.stop();
+111 -3
View File
@@ -1,4 +1,7 @@
import { layer } from "@fortawesome/fontawesome-svg-core";
import { VideoNode } from "./Nodes/VideoNode";
import Konva from "konva";
import { CanvasHTMLAttributes } from "react";
export class RenderEngine {
private videoNodes: VideoNode[];
private offScreenCanvas: OffscreenCanvas;
@@ -6,12 +9,71 @@ export class RenderEngine {
private isProcessing: boolean;
private frames: ImageBitmap[];
constructor(offScreenCanvas: OffscreenCanvas) {
// capturing composite within window
private videoLayer: Konva.Layer;
private videoDrawCanvas: HTMLCanvasElement;
private videoDrawContext: CanvasRenderingContext2D | null;
constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) {
this.videoNodes = [];
this.offScreenCanvas = offScreenCanvas;
this.context = this.offScreenCanvas.getContext("2d");
this.isProcessing = false;
this.frames = [];
// Rendering surface resolution
this.videoLayer = videoLayer;
var positionX = window.innerWidth / 2 - 720 / 2;
var positionY = window.innerHeight / 2 - 1080 / 2;
// https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
var width = 720;
var height = 1280;
const captureCanvas = new Konva.Rect({
x: positionX,
y: positionY,
width: width,
height: height,
fill: "white",
stroke: "black",
strokeWidth: 1,
draggable: false,
});
this.videoLayer.add(captureCanvas);
// set furest back
captureCanvas.setZIndex(-10);
// get the render position on the layer
this.videoDrawCanvas = this.renderPortionOfLayer(
this.videoLayer,
positionX,
positionY,
width,
height,
);
// sadly exists as a NOT AS a offscreen rendering context. ? maybe bitmaprenderer ?
this.videoDrawContext = this.videoDrawCanvas.getContext("2d");
}
// This function uses a portion of the video layer to capture just the capture canvas.
// capture everything after seeking each video node.
renderPortionOfLayer(
layer: Konva.Layer,
x: number,
y: number,
width: number,
height: number,
): HTMLCanvasElement {
const canvas = layer.toCanvas({
x: x,
y: y,
width: width,
height: height,
});
return canvas;
}
private findLongestVideoLength(): number {
@@ -62,9 +124,13 @@ export class RenderEngine {
this.videoNodes.forEach((item: VideoNode) => {});
// find the longest video
// find the longest video node;
const numberOfFrames = this.findLongestVideoLength();
await this.processFrame();
await this.render(numberOfFrames);
// only to test video node
//await this.processFrame();
}
public stopProcessing() {
@@ -76,6 +142,48 @@ export class RenderEngine {
return frameNumber / frameRate;
}
// TODO render loop should be.
// find longest video
// then seek through each node 1 step.
// stop ignore stepping if the duration is less.
private async render(numberOfFrames: number) {
if (!this.isProcessing) return;
for (let j = 0; j < numberOfFrames; j++) {
const videoNode = this.videoNodes[j];
videoNode.stop();
const numberOfFrames = videoNode.getNumberFrames();
for (let i = 0; i <= numberOfFrames; i++) {
const frameTime = this.calculateFrameTime(i, videoNode.fps);
console.log(i);
console.log(frameTime);
console.log(videoNode.duration);
if (frameTime < videoNode.duration) {
await videoNode.seek(frameTime);
this.offScreenCanvas.width = videoNode.node.getSize().width;
this.offScreenCanvas.height = videoNode.node.getSize().height;
if (this.context) {
this.context.drawImage(
videoNode.videoComponent,
0,
0,
this.offScreenCanvas.width,
this.offScreenCanvas.height,
);
this.frames.push(this.offScreenCanvas.transferToImageBitmap());
}
} // end of if
} // end of for.
}
}
//DO NOT USE: This code is to test whether or not we can seek frames on each video nodes.
//To check the correctness of the seeking.
private async processFrame() {
if (!this.isProcessing) return;