mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-15 22:32:13 +00:00
Have to test the new canvas bounds render technique.
This commit is contained in:
1 parent
9424e272e2
commit
aafa6fdb60
2 files changed
+124
-129
No files matched your search
+13
-126
@@ -3,7 +3,7 @@ import { VideoNode } from "./Nodes/VideoNode";
|
||||
import { uiAccess } from "~/signals";
|
||||
import { uiEvents } from "~/signals";
|
||||
import { RenderEngine } from "./RenderEngine";
|
||||
import { layer } from "@fortawesome/fontawesome-svg-core";
|
||||
|
||||
export class Engine {
|
||||
private canvasReference: HTMLDivElement;
|
||||
private stage: Konva.Stage;
|
||||
@@ -19,6 +19,9 @@ export class Engine {
|
||||
width: window.innerWidth,
|
||||
height: window.innerHeight,
|
||||
});
|
||||
const videoLayer = new Konva.Layer();
|
||||
this.videoLayer = videoLayer;
|
||||
this.stage.add(videoLayer);
|
||||
|
||||
// Listen to changes in container size
|
||||
const resizeObserver = new ResizeObserver(() => {
|
||||
@@ -28,12 +31,10 @@ export class Engine {
|
||||
resizeObserver.observe(this.canvasReference);
|
||||
this.applyChanges();
|
||||
|
||||
const videoLayer = new Konva.Layer();
|
||||
this.videoLayer = videoLayer;
|
||||
this.stage.add(videoLayer);
|
||||
// core layer for all the work done.
|
||||
|
||||
this.offScreenCanvas = new OffscreenCanvas(0, 0);
|
||||
const context = this.offScreenCanvas.getContext("2d");
|
||||
this.renderEngine = new RenderEngine(this.videoLayer, this.offScreenCanvas);
|
||||
|
||||
uiEvents.onGetStagedImage((image) => {
|
||||
this.addImage(image);
|
||||
@@ -54,7 +55,7 @@ export class Engine {
|
||||
}
|
||||
|
||||
public initializeStage(sceneToken: string) {
|
||||
// load canvas that was originaly saved
|
||||
// load canvas that was originaly saved TODO Save manager for resharing.
|
||||
|
||||
uiAccess.imageToolbar.hide();
|
||||
uiAccess.loadingBar.hide();
|
||||
@@ -92,101 +93,21 @@ export class Engine {
|
||||
"https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4",
|
||||
);
|
||||
|
||||
const node2 = new VideoNode(
|
||||
"",
|
||||
this.offScreenCanvas,
|
||||
this.videoLayer,
|
||||
1600,
|
||||
700,
|
||||
"https://storage.googleapis.com/vocodes-public/media/r/q/p/r/e/rqpret6mkh18dqwjqwghhdqf15x720s1/storyteller_rqpret6mkh18dqwjqwghhdqf15x720s1.mp4",
|
||||
);
|
||||
// CODE TO TEST RENDER ENGINE
|
||||
// Testing render engine
|
||||
this.renderEngine.addNodes(videoNode);
|
||||
|
||||
var positionX = window.innerWidth / 2 - 720 / 2;
|
||||
var positionY = window.innerHeight / 2 - 1080 / 2;
|
||||
await this.renderEngine.render();
|
||||
|
||||
var width = 720;
|
||||
var height = 1080;
|
||||
|
||||
const activeFrame = new Konva.Rect({
|
||||
x: positionX,
|
||||
y: positionY,
|
||||
width: width,
|
||||
height: height,
|
||||
fill: "white",
|
||||
stroke: "black",
|
||||
strokeWidth: 1,
|
||||
draggable: false,
|
||||
});
|
||||
|
||||
const activeFrame2 = new Konva.Rect({
|
||||
x: positionX,
|
||||
y: positionY,
|
||||
width: 100,
|
||||
height: 100,
|
||||
fill: "green",
|
||||
stroke: "black",
|
||||
strokeWidth: 1,
|
||||
draggable: false,
|
||||
});
|
||||
|
||||
this.videoLayer.add(activeFrame);
|
||||
this.videoLayer.add(activeFrame2);
|
||||
activeFrame2.moveToTop();
|
||||
activeFrame.moveToBottom();
|
||||
|
||||
await this.sleep(5000);
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const canvas = await this.renderPortionOfLayer(
|
||||
this.videoLayer,
|
||||
positionX,
|
||||
positionY,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
|
||||
canvas.toBlob((blob) => {
|
||||
if (blob) {
|
||||
// Do something with the blob, e.g., upload it or save it
|
||||
console.log("Blob created:", blob);
|
||||
// Example: Create a download link for the blob
|
||||
const link = document.createElement("a");
|
||||
link.href = URL.createObjectURL(blob);
|
||||
link.download = "canvas-output.png";
|
||||
link.click();
|
||||
} else {
|
||||
console.error("Failed to create blob");
|
||||
}
|
||||
}, "image/png");
|
||||
|
||||
await this.sleep(2000);
|
||||
}
|
||||
//this.renderEngine.addNodes(videoNode);
|
||||
|
||||
// Call this when
|
||||
// Call this when test video nodes
|
||||
//await this.renderEngine.startProcessing();
|
||||
|
||||
//videoNode.simulatedLoading();
|
||||
// TODO support Text nodes
|
||||
|
||||
this.videoLayer.add(textNode);
|
||||
}
|
||||
|
||||
async renderPortionOfLayer(
|
||||
layer: Konva.Layer,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
) {
|
||||
const canvas = layer.toCanvas({
|
||||
x: x,
|
||||
y: y,
|
||||
width: width,
|
||||
height: height,
|
||||
});
|
||||
return canvas;
|
||||
}
|
||||
|
||||
public addImage(imageFile: File) {
|
||||
// main API:
|
||||
const imageObj = new Image();
|
||||
@@ -210,37 +131,3 @@ export class Engine {
|
||||
console.log("addVideo", videoFile);
|
||||
}
|
||||
}
|
||||
|
||||
// do nothing, animation just need to update the layer
|
||||
// try {
|
||||
// if (!context) {
|
||||
// console.log("Context is dead.");
|
||||
// return;
|
||||
// }
|
||||
// offScreenCanvas.width = videoNode.getWidth();
|
||||
// offScreenCanvas.height = videoNode.getHeight();
|
||||
// context.drawImage(
|
||||
// video,
|
||||
// 0,
|
||||
// 0,
|
||||
// videoNode.getWidth(),
|
||||
// videoNode.getHeight(),
|
||||
// );
|
||||
// console.log("draw");
|
||||
// const blob = await offScreenCanvas.convertToBlob({
|
||||
// quality: 1.0,
|
||||
// type: "image/jpeg",
|
||||
// });
|
||||
// console.log("blob");
|
||||
// const link = document.createElement("a");
|
||||
// link.href = URL.createObjectURL(blob);
|
||||
// link.download = "canvas-output.jpg";
|
||||
// // Trigger the download
|
||||
// link.click();
|
||||
// // Clean up the URL object
|
||||
// URL.revokeObjectURL(link.href);
|
||||
// console.log("hello");
|
||||
// } catch (error) {
|
||||
// console.log(error);
|
||||
// }
|
||||
// anim.stop();
|
||||
@@ -1,4 +1,7 @@
|
||||
import { layer } from "@fortawesome/fontawesome-svg-core";
|
||||
import { VideoNode } from "./Nodes/VideoNode";
|
||||
import Konva from "konva";
|
||||
import { CanvasHTMLAttributes } from "react";
|
||||
export class RenderEngine {
|
||||
private videoNodes: VideoNode[];
|
||||
private offScreenCanvas: OffscreenCanvas;
|
||||
@@ -6,12 +9,71 @@ export class RenderEngine {
|
||||
private isProcessing: boolean;
|
||||
|
||||
private frames: ImageBitmap[];
|
||||
constructor(offScreenCanvas: OffscreenCanvas) {
|
||||
|
||||
// capturing composite within window
|
||||
private videoLayer: Konva.Layer;
|
||||
private videoDrawCanvas: HTMLCanvasElement;
|
||||
private videoDrawContext: CanvasRenderingContext2D | null;
|
||||
constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) {
|
||||
this.videoNodes = [];
|
||||
this.offScreenCanvas = offScreenCanvas;
|
||||
this.context = this.offScreenCanvas.getContext("2d");
|
||||
this.isProcessing = false;
|
||||
this.frames = [];
|
||||
|
||||
// Rendering surface resolution
|
||||
|
||||
this.videoLayer = videoLayer;
|
||||
|
||||
var positionX = window.innerWidth / 2 - 720 / 2;
|
||||
var positionY = window.innerHeight / 2 - 1080 / 2;
|
||||
// https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
|
||||
var width = 720;
|
||||
var height = 1280;
|
||||
|
||||
const captureCanvas = new Konva.Rect({
|
||||
x: positionX,
|
||||
y: positionY,
|
||||
width: width,
|
||||
height: height,
|
||||
fill: "white",
|
||||
stroke: "black",
|
||||
strokeWidth: 1,
|
||||
draggable: false,
|
||||
});
|
||||
this.videoLayer.add(captureCanvas);
|
||||
// set furest back
|
||||
captureCanvas.setZIndex(-10);
|
||||
|
||||
// get the render position on the layer
|
||||
this.videoDrawCanvas = this.renderPortionOfLayer(
|
||||
this.videoLayer,
|
||||
positionX,
|
||||
positionY,
|
||||
width,
|
||||
height,
|
||||
);
|
||||
|
||||
// sadly exists as a NOT AS a offscreen rendering context. ? maybe bitmaprenderer ?
|
||||
this.videoDrawContext = this.videoDrawCanvas.getContext("2d");
|
||||
}
|
||||
|
||||
// This function uses a portion of the video layer to capture just the capture canvas.
|
||||
// capture everything after seeking each video node.
|
||||
renderPortionOfLayer(
|
||||
layer: Konva.Layer,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
): HTMLCanvasElement {
|
||||
const canvas = layer.toCanvas({
|
||||
x: x,
|
||||
y: y,
|
||||
width: width,
|
||||
height: height,
|
||||
});
|
||||
return canvas;
|
||||
}
|
||||
|
||||
private findLongestVideoLength(): number {
|
||||
@@ -62,9 +124,13 @@ export class RenderEngine {
|
||||
|
||||
this.videoNodes.forEach((item: VideoNode) => {});
|
||||
|
||||
// find the longest video
|
||||
// find the longest video node;
|
||||
const numberOfFrames = this.findLongestVideoLength();
|
||||
await this.processFrame();
|
||||
|
||||
await this.render(numberOfFrames);
|
||||
|
||||
// only to test video node
|
||||
//await this.processFrame();
|
||||
}
|
||||
|
||||
public stopProcessing() {
|
||||
@@ -76,6 +142,48 @@ export class RenderEngine {
|
||||
return frameNumber / frameRate;
|
||||
}
|
||||
|
||||
// TODO render loop should be.
|
||||
// find longest video
|
||||
// then seek through each node 1 step.
|
||||
// stop ignore stepping if the duration is less.
|
||||
private async render(numberOfFrames: number) {
|
||||
if (!this.isProcessing) return;
|
||||
for (let j = 0; j < numberOfFrames; j++) {
|
||||
const videoNode = this.videoNodes[j];
|
||||
|
||||
videoNode.stop();
|
||||
const numberOfFrames = videoNode.getNumberFrames();
|
||||
|
||||
for (let i = 0; i <= numberOfFrames; i++) {
|
||||
const frameTime = this.calculateFrameTime(i, videoNode.fps);
|
||||
|
||||
console.log(i);
|
||||
console.log(frameTime);
|
||||
console.log(videoNode.duration);
|
||||
|
||||
if (frameTime < videoNode.duration) {
|
||||
await videoNode.seek(frameTime);
|
||||
this.offScreenCanvas.width = videoNode.node.getSize().width;
|
||||
this.offScreenCanvas.height = videoNode.node.getSize().height;
|
||||
|
||||
if (this.context) {
|
||||
this.context.drawImage(
|
||||
videoNode.videoComponent,
|
||||
0,
|
||||
0,
|
||||
this.offScreenCanvas.width,
|
||||
this.offScreenCanvas.height,
|
||||
);
|
||||
|
||||
this.frames.push(this.offScreenCanvas.transferToImageBitmap());
|
||||
}
|
||||
} // end of if
|
||||
} // end of for.
|
||||
}
|
||||
}
|
||||
|
||||
//DO NOT USE: This code is to test whether or not we can seek frames on each video nodes.
|
||||
//To check the correctness of the seeking.
|
||||
private async processFrame() {
|
||||
if (!this.isProcessing) return;
|
||||
|
||||
|
||||
Reference in new issue
Block a user