From 63ca9b5fa3bb46bef450067e1bfb787e5c0c37a2 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 12 Sep 2024 23:49:01 -0400 Subject: [PATCH] Trying the bitmap transfer method, since canvas transfer doesn't actually work in chrome lol. --- src/KonvaApp/Engine.ts | 70 ++------ .../RenderingPrimitives/RenderEngine.ts | 152 +++++++++--------- .../Diffusion/DiffusionSharedWorker.ts | 60 ++++++- .../Diffusion/DiffusionSharedWorkerClient.ts | 17 +- src/KonvaApp/index.ts | 4 +- 5 files changed, 162 insertions(+), 141 deletions(-) diff --git a/src/KonvaApp/Engine.ts b/src/KonvaApp/Engine.ts index 0a16826d51..c95025e02d 100644 --- a/src/KonvaApp/Engine.ts +++ b/src/KonvaApp/Engine.ts @@ -5,12 +5,6 @@ import { uiEvents } from "~/signals"; import { RenderEngine } from "./RenderingPrimitives/RenderEngine"; import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum"; -import { - WorkFunction, - ProgressData, - WorkQueue, -} from "./WorkerPrimitives/GenericWorker"; -import { SharedWorkerClient } from "./WorkerPrimitives/SharedWorkerClient"; export class Engine { private canvasReference: HTMLDivElement; @@ -66,7 +60,20 @@ export class Engine { uiEvents.toolbarMain.SELECT_ONE.onClick(() => { console.log("select one is clicked"); }); + uiEvents.toolbarMain.SAVE.onClick(async (event) => { + uiAccess.toolbarMain.changeButtonState( + ToolbarMainButtonNames.AI_STYLIZE, + { disabled: true }, + ); + console.log("onClick"); + await this.renderEngine.startProcessing(); + + uiAccess.toolbarMain.changeButtonState( + ToolbarMainButtonNames.AI_STYLIZE, + { disabled: false }, + ); + }); uiEvents.toolbarMain.AI_STYLIZE.onClick(async (event) => { uiAccess.toolbarMain.changeButtonState( ToolbarMainButtonNames.AI_STYLIZE, @@ -77,7 +84,7 @@ export class Engine { "SLEEP", `${sleepytstart.getMinutes()}:${sleepytstart.getSeconds()}`, ); - await this.renderEngine.startProcessing(); + const sleeptend = new Date(); console.log( "DONE", @@ -102,54 +109,7 @@ export class Engine { } // Sandbox is quickly a way to test your idea. - public sandbox() { - // How to use: - // const exampleWorkFunction: WorkFunction = async ( - // data: number, - // reportProgress: (progress: number) => void, - // ) => { - // // Simulate some asynchronous work with progress reporting - // for (let i = 0; i <= 100; i += 20) { - // await new Promise((resolve) => setTimeout(resolve, 200)); - // reportProgress(i); - // } - // return data * 2; - // }; - - // const progressCallback = (progressData: ProgressData) => { - // console.log( - // `Job ${progressData.jobId} progress: ${progressData.progress}%`, - // ); - // }; - - // const workQueue = new WorkQueue( - // exampleWorkFunction, - // progressCallback, - // ); - // workQueue.addWork(1, 10); - // workQueue.addWork(2, 20); - // workQueue.addWork(3, 30); - - // const sharedWorker = new SharedWorker( - // "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts", - // { - // type: "module", - // }, - // ); - - // // Get the port for communication - // let port = sharedWorker.port; - // port.start(); - // // Set up the message event listener - // port.onmessage = this.onMessage.bind(this); - // port.postMessage({ jobID: 1, data: 2, isDoneStreaming: false }); - - const value = new SharedWorkerClient( - "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts", - undefined, - ); - value.send({ jobID: 1, data: 1, isDoneStreaming: false }); - } + public sandbox() {} public onMessage(event: MessageEvent) { console.log("Message From Shared Worker"); diff --git a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts index 7ad04d9c95..591e4fc658 100644 --- a/src/KonvaApp/RenderingPrimitives/RenderEngine.ts +++ b/src/KonvaApp/RenderingPrimitives/RenderEngine.ts @@ -1,7 +1,18 @@ import { VideoNode } from "../Nodes/VideoNode"; import Konva from "konva"; +import { RenderTask } from "./RenderTask"; -import { RenderTask, WorkerEvent } from "./RenderTask"; +import { DiffusionSharedWorkerClient } from "../SharedWorkers/Diffusion/DiffusionSharedWorkerClient"; +import { + SharedWorkerRequest, + SharedWorkerResponse, +} from "../WorkerPrimitives/SharedWorkerBase"; +import { + DiffusionSharedWorkerProgressData, + DiffusionSharedWorkerResponseData, + DiffusionSharedWorker, + DiffusionSharedWorkerItemData, +} from "../SharedWorkers/Diffusion/DiffusionSharedWorker"; // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio export class RenderEngine { @@ -9,7 +20,6 @@ export class RenderEngine { private offScreenCanvas: OffscreenCanvas; private context: OffscreenCanvasRenderingContext2D | null; - private offscreenCanvasCache: OffscreenCanvas[] = []; private isProcessing: boolean; private frames: ImageBitmap[]; @@ -23,15 +33,23 @@ export class RenderEngine { private positionY: number; private canUseSharedWorker: boolean; - private sharedWorker: SharedWorker | undefined; + private port: MessagePort | undefined; private maxFrames: number; + + private diffusionWorker: + | DiffusionSharedWorkerClient< + DiffusionSharedWorkerItemData, + DiffusionSharedWorkerResponseData, + DiffusionSharedWorkerProgressData + > + | undefined; + constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) { this.videoNodes = []; - this.isProcessing = false; - // TODO make this dynamic + // TODO: Make this dynamic and update this on change of canvas. this.width = 720; this.height = 1280; @@ -61,42 +79,29 @@ export class RenderEngine { this.maxFrames = 7 * 24; - // DEBUG ONLY - // const rectangle = new Konva.Rect({ - // x: this.positionX, - // y: this.positionY, - // width: 100, - // height: 100, - // fill: "green", - // stroke: "black", - // strokeWidth: 1, - // draggable: false, - // }); - //this.videoLayer.add(rectangle); this.videoLayer.add(captureCanvas); - // set back + // send back captureCanvas.setZIndex(0); - this.sharedWorker = undefined; this.canUseSharedWorker = false; this.setupSharedWorker(); + + //this.debug(); } - // Manage the worker - onMessage(event: MessageEvent) { - const message = event.data; - console.log("Main script received:", message); - // Send a response back to the worker - if (!this.port) { - return console.log("Undefined Worker"); - } - } - - sendMessage() { - if (!this.port) { - return console.log("Undefined Worker"); - } - this.port.postMessage("Hello from Main Script"); + debug() { + // DEBUG ONLY + const rectangle = new Konva.Rect({ + x: this.positionX, + y: this.positionY, + width: 100, + height: 100, + fill: "green", + stroke: "black", + strokeWidth: 1, + draggable: false, + }); + this.videoLayer.add(rectangle); } async sendCanvasPayload(renderTask: RenderTask) { @@ -106,31 +111,24 @@ export class RenderEngine { this.port.postMessage(renderTask); } + onMessageReceived( + response: SharedWorkerResponse< + DiffusionSharedWorkerResponseData, + DiffusionSharedWorkerProgressData + >, + ) { + console.log(response); + } + setupSharedWorker() { if (typeof SharedWorker !== "undefined") { - this.offscreenCanvasCache = []; - // canvas size should be defined here - for (let i = 0; i < this.maxFrames; i++) { - let offScreenCanvas = new OffscreenCanvas(this.width, this.height); - this.offscreenCanvasCache.push(offScreenCanvas); - } - console.log("Shared Workers are supported in this browser."); // Debug chrome://inspect/#workers - // TODO ensure that the cors is solved for this. - this.sharedWorker = new SharedWorker( - "src\\KonvaApp\\RenderingPrimitives\\RenderSharedWorker.ts", - { - type: "module", - }, + this.diffusionWorker = new DiffusionSharedWorkerClient( + "src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts", + this.onMessageReceived.bind(this), ); - // Get the port for communication - this.port = this.sharedWorker.port; - this.port.start(); - // Set up the message event listener - this.port.onmessage = this.onMessage.bind(this); - this.canUseSharedWorker = true; } else { console.log("Shared Workers are not supported in this browser."); @@ -253,7 +251,6 @@ export class RenderEngine { for (let j = 0; j < largestNumberOfFrames; j++) { // Seek Video Nodes first then draw - let frameTime = undefined; for (let i = 0; i < this.videoNodes.length; i++) { @@ -300,10 +297,19 @@ export class RenderEngine { } // end of for each frame } else { console.log("Using Shared Worker"); - const canvas = this.offscreenCanvasCache[j]; - const context = canvas.getContext("2d"); + // decode on the shared webworker. - context?.drawImage( + if (!this.context) { + console.log("Context Didn't Initialize"); + return; + } + + if (!this.diffusionWorker) { + console.log("Didnt Initialize Diffusion"); + return; + } + + this.context.drawImage( this.videoLayer.canvas._canvas, this.positionX, this.positionY, @@ -315,24 +321,26 @@ export class RenderEngine { this.height, ); - this.sendCanvasPayload({ - jobID: 1, - isDone: false, - event: WorkerEvent.DATA, - data: { - offscreenCanvas: canvas, - id: j, - }, - }); + const data: DiffusionSharedWorkerItemData = { + height: this.height, + width: this.width, + imageBitmap: this.offScreenCanvas.transferToImageBitmap(), + frame: j, + totalFrames: this.maxFrames, + }; + + let isDoneStreaming = false; if (j == this.maxFrames - 1) { - this.sendCanvasPayload({ - jobID: 1, - isDone: true, - event: WorkerEvent.DATA, - data: undefined, - }); + isDoneStreaming = true; } + + const container: SharedWorkerRequest = { + data: data, + isDoneStreaming: isDoneStreaming, + jobID: 1, + }; + this.diffusionWorker.send(container); } } } diff --git a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts index db5ed3883a..1ffae83ff1 100644 --- a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts +++ b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts @@ -1,4 +1,7 @@ // Example of this working. +import { error } from "@techstark/opencv-js"; +import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js"; + import { ProgressData, WorkResult, @@ -16,11 +19,16 @@ export interface DiffusionSharedWorkerProgressData { } export interface DiffusionSharedWorkerItemData { - canvas: OffscreenCanvas; + imageBitmap: ImageBitmap; + totalFrames: number; + frame: number; + height: number; + width: number; } export interface DiffusionSharedWorkerResponseData { videoUrl: string; + zipBlob: Blob; // debug purposes } export class DiffusionSharedWorker extends SharedWorkerBase< @@ -28,9 +36,29 @@ export class DiffusionSharedWorker extends SharedWorkerBase< DiffusionSharedWorkerResponseData, DiffusionSharedWorkerProgressData > { + public zipFileWriter: BlobWriter; + public zipWriter: ZipWriter; + public imageType: string; + public totalFrames: number; + + public offscreenCanvas: OffscreenCanvas | undefined; + public bitmapContext: ImageBitmapRenderingContext | undefined | null; + constructor(port: MessagePort) { super(port); this.setup(this.workFunction.bind(this), this.progressFunction.bind(this)); + this.offscreenCanvas = undefined; + this.bitmapContext = undefined; + this.imageType = "image/jpeg"; + this.zipFileWriter = new BlobWriter(this.imageType); + this.zipWriter = new ZipWriter(this.zipFileWriter); + this.totalFrames = 0; + } + + async reset() { + this.zipFileWriter = new BlobWriter(this.imageType); + this.zipWriter = new ZipWriter(this.zipFileWriter); + this.totalFrames = 0; } // Data here will be shipped off for progressive loading async workFunction( @@ -41,19 +69,41 @@ export class DiffusionSharedWorker extends SharedWorkerBase< data: DiffusionSharedWorkerProgressData, ) => void, ): Promise<[DiffusionSharedWorkerResponseData | undefined, boolean]> { - console.log(`Working Item ${item}`); + if (this.offscreenCanvas === undefined) { + this.offscreenCanvas = new OffscreenCanvas(item.width, item.height); + this.bitmapContext = this.offscreenCanvas.getContext("bitmaprenderer"); + } + if (!this.bitmapContext) { + console.log("Failed to create bitmap context."); + throw Error("Bitmap Rendering Context Not Availible."); + } + + this.bitmapContext.transferFromImageBitmap(item.imageBitmap); + + const blob = await this.offscreenCanvas.convertToBlob({ + quality: 1.0, + type: this.imageType, + }); + + const aproxSteps = item.totalFrames * 2; const progressData: DiffusionSharedWorkerProgressData = { url: "" }; - reportProgress(0, progressData); - // check if stream is done. + reportProgress(item.frame / aproxSteps, progressData); + + const name = String(item.frame).padStart(5, "0"); // '0009' + await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob)); if (isDoneStreaming === false) { return [undefined, false]; } // make request via api with options - const responseData: DiffusionSharedWorkerResponseData = { videoUrl: "" }; + const zipBlob = await this.zipWriter.close(); + const responseData: DiffusionSharedWorkerResponseData = { + videoUrl: "", + zipBlob: zipBlob, + }; return [responseData, true]; } diff --git a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts index e1e313d7f6..d65ffdebfc 100644 --- a/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts +++ b/src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts @@ -1,5 +1,8 @@ -import { SharedWorkerRequest, SharedWorkerResponse } from "./SharedWorkerBase"; -import { ResponseType } from "./SharedWorkerBase"; +import { + SharedWorkerRequest, + SharedWorkerResponse, +} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase"; +import { ResponseType } from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase"; import { DiffusionSharedWorkerItemData, @@ -29,7 +32,7 @@ export class DiffusionSharedWorkerClient< >, ) => void, ) { - // exampl + // example // "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts" this.sharedWorker = new SharedWorker(workerPath, { type: "module", @@ -47,13 +50,13 @@ export class DiffusionSharedWorkerClient< DiffusionSharedWorkerItemData >; if (data.responseType === ResponseType.error) { - console.log(`TemplateSharedWorkerClient Error:${data}`); + console.log(`DiffusionSharedWorkerClient Error:${data}`); } else if (data.responseType === ResponseType.progress) { - console.log(`TemplateSharedWorkerClient Progress:${data}`); + console.log(`DiffusionSharedWorkerClient Progress:${data}`); } else if (data.responseType === ResponseType.result) { - console.log(`TemplateSharedWorkerClient Result:${data}`); + console.log(`DiffusionSharedWorkerClient Result:${data}`); } else { - console.log("TemplateSharedWorkerClient Message Unknown"); + console.log(`DiffusionSharedWorkerClient Message Unknown ${data}`); } console.log(`TemplateSharedWorker Response: ${JSON.stringify(event.data)}`); } diff --git a/src/KonvaApp/index.ts b/src/KonvaApp/index.ts index 80a47e46e5..ed1c1ec111 100644 --- a/src/KonvaApp/index.ts +++ b/src/KonvaApp/index.ts @@ -4,8 +4,8 @@ export { Engine as EngineType }; export const KonvaApp = (element: HTMLDivElement) => { const engine = new Engine(element); - engine.sandbox(); - //engine.initializeStage(""); + //engine.sandbox(); + engine.initializeStage(""); return engine; };