daniel543x/artcraft-services
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trying the bitmap transfer method, since canvas transfer

Browse Source 
doesn't actually work in chrome lol.
This commit is contained in:
Michael Chung committed 2024-09-12 23:49:01 -04:00
1 parent 08caa191b2
commit 63ca9b5fa3
5 files changed
+162 -141
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/KonvaApp/Engine.ts
+15 -55
View File
@@ -5,12 +5,6 @@ import { uiEvents } from "~/signals";
import { RenderEngine } from "./RenderingPrimitives/RenderEngine";
import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
import {
WorkFunction,
ProgressData,
WorkQueue,
} from "./WorkerPrimitives/GenericWorker";
import { SharedWorkerClient } from "./WorkerPrimitives/SharedWorkerClient";
export class Engine {
private canvasReference: HTMLDivElement;
@@ -66,7 +60,20 @@ export class Engine {
uiEvents.toolbarMain.SELECT_ONE.onClick(() => {
console.log("select one is clicked");
});
uiEvents.toolbarMain.SAVE.onClick(async (event) => {
uiAccess.toolbarMain.changeButtonState(
ToolbarMainButtonNames.AI_STYLIZE,
{ disabled: true },
);
console.log("onClick");
await this.renderEngine.startProcessing();
uiAccess.toolbarMain.changeButtonState(
ToolbarMainButtonNames.AI_STYLIZE,
{ disabled: false },
);
});
uiEvents.toolbarMain.AI_STYLIZE.onClick(async (event) => {
uiAccess.toolbarMain.changeButtonState(
ToolbarMainButtonNames.AI_STYLIZE,
@@ -77,7 +84,7 @@ export class Engine {
"SLEEP",
`${sleepytstart.getMinutes()}:${sleepytstart.getSeconds()}`,
);
await this.renderEngine.startProcessing();
const sleeptend = new Date();
console.log(
"DONE",
@@ -102,54 +109,7 @@ export class Engine {
}
// Sandbox is quickly a way to test your idea.
public sandbox() {
// How to use:
// const exampleWorkFunction: WorkFunction<number, number> = async (
// data: number,
// reportProgress: (progress: number) => void,
// ) => {
// // Simulate some asynchronous work with progress reporting
// for (let i = 0; i <= 100; i += 20) {
// await new Promise((resolve) => setTimeout(resolve, 200));
// reportProgress(i);
// }
// return data * 2;
// };
// const progressCallback = (progressData: ProgressData) => {
// console.log(
// `Job ${progressData.jobId} progress: ${progressData.progress}%`,
// );
// };
// const workQueue = new WorkQueue<number, number>(
// exampleWorkFunction,
// progressCallback,
// );
// workQueue.addWork(1, 10);
// workQueue.addWork(2, 20);
// workQueue.addWork(3, 30);
// const sharedWorker = new SharedWorker(
// "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts",
// {
// type: "module",
// },
// );
// // Get the port for communication
// let port = sharedWorker.port;
// port.start();
// // Set up the message event listener
// port.onmessage = this.onMessage.bind(this);
// port.postMessage({ jobID: 1, data: 2, isDoneStreaming: false });
const value = new SharedWorkerClient<number, number, number>(
"src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts",
undefined,
);
value.send({ jobID: 1, data: 1, isDoneStreaming: false });
}
public sandbox() {}
public onMessage(event: MessageEvent) {
console.log("Message From Shared Worker");
src/KonvaApp/RenderingPrimitives/RenderEngine.ts
+80 -72
View File
@@ -1,7 +1,18 @@
import { VideoNode } from "../Nodes/VideoNode";
import Konva from "konva";
import { RenderTask } from "./RenderTask";
import { RenderTask, WorkerEvent } from "./RenderTask";
import { DiffusionSharedWorkerClient } from "../SharedWorkers/Diffusion/DiffusionSharedWorkerClient";
import {
SharedWorkerRequest,
SharedWorkerResponse,
} from "../WorkerPrimitives/SharedWorkerBase";
import {
DiffusionSharedWorkerProgressData,
DiffusionSharedWorkerResponseData,
DiffusionSharedWorker,
DiffusionSharedWorkerItemData,
} from "../SharedWorkers/Diffusion/DiffusionSharedWorker";
// https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
export class RenderEngine {
@@ -9,7 +20,6 @@ export class RenderEngine {
private offScreenCanvas: OffscreenCanvas;
private context: OffscreenCanvasRenderingContext2D | null;
private offscreenCanvasCache: OffscreenCanvas[] = [];
private isProcessing: boolean;
private frames: ImageBitmap[];
@@ -23,15 +33,23 @@ export class RenderEngine {
private positionY: number;
private canUseSharedWorker: boolean;
private sharedWorker: SharedWorker | undefined;
private port: MessagePort | undefined;
private maxFrames: number;
private diffusionWorker:
| DiffusionSharedWorkerClient<
DiffusionSharedWorkerItemData,
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>
| undefined;
constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) {
this.videoNodes = [];
this.isProcessing = false;
// TODO make this dynamic
// TODO: Make this dynamic and update this on change of canvas.
this.width = 720;
this.height = 1280;
@@ -61,42 +79,29 @@ export class RenderEngine {
this.maxFrames = 7 * 24;
// DEBUG ONLY
// const rectangle = new Konva.Rect({
// x: this.positionX,
// y: this.positionY,
// width: 100,
// height: 100,
// fill: "green",
// stroke: "black",
// strokeWidth: 1,
// draggable: false,
// });
//this.videoLayer.add(rectangle);
this.videoLayer.add(captureCanvas);
// set back
// send back
captureCanvas.setZIndex(0);
this.sharedWorker = undefined;
this.canUseSharedWorker = false;
this.setupSharedWorker();
//this.debug();
}
// Manage the worker
onMessage(event: MessageEvent) {
const message = event.data;
console.log("Main script received:", message);
// Send a response back to the worker
if (!this.port) {
return console.log("Undefined Worker");
}
}
sendMessage() {
if (!this.port) {
return console.log("Undefined Worker");
}
this.port.postMessage("Hello from Main Script");
debug() {
// DEBUG ONLY
const rectangle = new Konva.Rect({
x: this.positionX,
y: this.positionY,
width: 100,
height: 100,
fill: "green",
stroke: "black",
strokeWidth: 1,
draggable: false,
});
this.videoLayer.add(rectangle);
}
async sendCanvasPayload(renderTask: RenderTask) {
@@ -106,31 +111,24 @@ export class RenderEngine {
this.port.postMessage(renderTask);
}
onMessageReceived(
response: SharedWorkerResponse<
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
>,
) {
console.log(response);
}
setupSharedWorker() {
if (typeof SharedWorker !== "undefined") {
this.offscreenCanvasCache = [];
// canvas size should be defined here
for (let i = 0; i < this.maxFrames; i++) {
let offScreenCanvas = new OffscreenCanvas(this.width, this.height);
this.offscreenCanvasCache.push(offScreenCanvas);
}
console.log("Shared Workers are supported in this browser.");
// Debug chrome://inspect/#workers
// TODO ensure that the cors is solved for this.
this.sharedWorker = new SharedWorker(
"src\\KonvaApp\\RenderingPrimitives\\RenderSharedWorker.ts",
{
type: "module",
},
this.diffusionWorker = new DiffusionSharedWorkerClient(
"src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts",
this.onMessageReceived.bind(this),
);
// Get the port for communication
this.port = this.sharedWorker.port;
this.port.start();
// Set up the message event listener
this.port.onmessage = this.onMessage.bind(this);
this.canUseSharedWorker = true;
} else {
console.log("Shared Workers are not supported in this browser.");
@@ -253,7 +251,6 @@ export class RenderEngine {
for (let j = 0; j < largestNumberOfFrames; j++) {
// Seek Video Nodes first then draw
let frameTime = undefined;
for (let i = 0; i < this.videoNodes.length; i++) {
@@ -300,10 +297,19 @@ export class RenderEngine {
} // end of for each frame
} else {
console.log("Using Shared Worker");
const canvas = this.offscreenCanvasCache[j];
const context = canvas.getContext("2d");
// decode on the shared webworker.
context?.drawImage(
if (!this.context) {
console.log("Context Didn't Initialize");
return;
}
if (!this.diffusionWorker) {
console.log("Didnt Initialize Diffusion");
return;
}
this.context.drawImage(
this.videoLayer.canvas._canvas,
this.positionX,
this.positionY,
@@ -315,24 +321,26 @@ export class RenderEngine {
this.height,
);
this.sendCanvasPayload({
jobID: 1,
isDone: false,
event: WorkerEvent.DATA,
data: {
offscreenCanvas: canvas,
id: j,
},
});
const data: DiffusionSharedWorkerItemData = {
height: this.height,
width: this.width,
imageBitmap: this.offScreenCanvas.transferToImageBitmap(),
frame: j,
totalFrames: this.maxFrames,
};
let isDoneStreaming = false;
if (j == this.maxFrames - 1) {
this.sendCanvasPayload({
jobID: 1,
isDone: true,
event: WorkerEvent.DATA,
data: undefined,
});
isDoneStreaming = true;
}
const container: SharedWorkerRequest<DiffusionSharedWorkerItemData> = {
data: data,
isDoneStreaming: isDoneStreaming,
jobID: 1,
};
this.diffusionWorker.send(container);
}
}
}
src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts
+55 -5
View File
@@ -1,4 +1,7 @@
// Example of this working.
import { error } from "@techstark/opencv-js";
import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
import {
ProgressData,
WorkResult,
@@ -16,11 +19,16 @@ export interface DiffusionSharedWorkerProgressData {
}
export interface DiffusionSharedWorkerItemData {
canvas: OffscreenCanvas;
imageBitmap: ImageBitmap;
totalFrames: number;
frame: number;
height: number;
width: number;
}
export interface DiffusionSharedWorkerResponseData {
videoUrl: string;
zipBlob: Blob; // debug purposes
}
export class DiffusionSharedWorker extends SharedWorkerBase<
@@ -28,9 +36,29 @@ export class DiffusionSharedWorker extends SharedWorkerBase<
DiffusionSharedWorkerResponseData,
DiffusionSharedWorkerProgressData
> {
public zipFileWriter: BlobWriter;
public zipWriter: ZipWriter<Blob>;
public imageType: string;
public totalFrames: number;
public offscreenCanvas: OffscreenCanvas | undefined;
public bitmapContext: ImageBitmapRenderingContext | undefined | null;
constructor(port: MessagePort) {
super(port);
this.setup(this.workFunction.bind(this), this.progressFunction.bind(this));
this.offscreenCanvas = undefined;
this.bitmapContext = undefined;
this.imageType = "image/jpeg";
this.zipFileWriter = new BlobWriter(this.imageType);
this.zipWriter = new ZipWriter(this.zipFileWriter);
this.totalFrames = 0;
}
async reset() {
this.zipFileWriter = new BlobWriter(this.imageType);
this.zipWriter = new ZipWriter(this.zipFileWriter);
this.totalFrames = 0;
}
// Data here will be shipped off for progressive loading
async workFunction(
@@ -41,19 +69,41 @@ export class DiffusionSharedWorker extends SharedWorkerBase<
data: DiffusionSharedWorkerProgressData,
) => void,
): Promise<[DiffusionSharedWorkerResponseData | undefined, boolean]> {
console.log(`Working Item ${item}`);
if (this.offscreenCanvas === undefined) {
this.offscreenCanvas = new OffscreenCanvas(item.width, item.height);
this.bitmapContext = this.offscreenCanvas.getContext("bitmaprenderer");
}
if (!this.bitmapContext) {
console.log("Failed to create bitmap context.");
throw Error("Bitmap Rendering Context Not Availible.");
}
this.bitmapContext.transferFromImageBitmap(item.imageBitmap);
const blob = await this.offscreenCanvas.convertToBlob({
quality: 1.0,
type: this.imageType,
});
const aproxSteps = item.totalFrames * 2;
const progressData: DiffusionSharedWorkerProgressData = { url: "" };
reportProgress(0, progressData);
// check if stream is done.
reportProgress(item.frame / aproxSteps, progressData);
const name = String(item.frame).padStart(5, "0"); // '0009'
await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob));
if (isDoneStreaming === false) {
return [undefined, false];
}
// make request via api with options
const responseData: DiffusionSharedWorkerResponseData = { videoUrl: "" };
const zipBlob = await this.zipWriter.close();
const responseData: DiffusionSharedWorkerResponseData = {
videoUrl: "",
zipBlob: zipBlob,
};
return [responseData, true];
}
src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts
+10 -7
View File
@@ -1,5 +1,8 @@
import { SharedWorkerRequest, SharedWorkerResponse } from "./SharedWorkerBase";
import { ResponseType } from "./SharedWorkerBase";
import {
SharedWorkerRequest,
SharedWorkerResponse,
} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
import { ResponseType } from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
import {
DiffusionSharedWorkerItemData,
@@ -29,7 +32,7 @@ export class DiffusionSharedWorkerClient<
>,
) => void,
) {
// exampl
// example
// "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts"
this.sharedWorker = new SharedWorker(workerPath, {
type: "module",
@@ -47,13 +50,13 @@ export class DiffusionSharedWorkerClient<
DiffusionSharedWorkerItemData
>;
if (data.responseType === ResponseType.error) {
console.log(`TemplateSharedWorkerClient Error:${data}`);
console.log(`DiffusionSharedWorkerClient Error:${data}`);
} else if (data.responseType === ResponseType.progress) {
console.log(`TemplateSharedWorkerClient Progress:${data}`);
console.log(`DiffusionSharedWorkerClient Progress:${data}`);
} else if (data.responseType === ResponseType.result) {
console.log(`TemplateSharedWorkerClient Result:${data}`);
console.log(`DiffusionSharedWorkerClient Result:${data}`);
} else {
console.log("TemplateSharedWorkerClient Message Unknown");
console.log(`DiffusionSharedWorkerClient Message Unknown ${data}`);
}
console.log(`TemplateSharedWorker Response: ${JSON.stringify(event.data)}`);
}
src/KonvaApp/index.ts
+2 -2
View File
@@ -4,8 +4,8 @@ export { Engine as EngineType };
export const KonvaApp = (element: HTMLDivElement) => {
const engine = new Engine(element);
engine.sandbox();
//engine.initializeStage("");
//engine.sandbox();
engine.initializeStage("");
return engine;
};