Trying the bitmap transfer method, since canvas transfer

doesn't actually work in chrome lol.

src/KonvaApp/Engine.ts

@@ -5,12 +5,6 @@ import { uiEvents } from "~/signals";
 import { RenderEngine } from "./RenderingPrimitives/RenderEngine";
 
 import { ToolbarMainButtonNames } from "~/components/features/ToolbarMain/enum";
-import {
-  WorkFunction,
-  ProgressData,
-  WorkQueue,
-} from "./WorkerPrimitives/GenericWorker";
-import { SharedWorkerClient } from "./WorkerPrimitives/SharedWorkerClient";
 
 export class Engine {
   private canvasReference: HTMLDivElement;
@@ -66,7 +60,20 @@ export class Engine {
     uiEvents.toolbarMain.SELECT_ONE.onClick(() => {
       console.log("select one is clicked");
     });
+    uiEvents.toolbarMain.SAVE.onClick(async (event) => {
+      uiAccess.toolbarMain.changeButtonState(
+        ToolbarMainButtonNames.AI_STYLIZE,
+        { disabled: true },
+      );
 
+      console.log("onClick");
+      await this.renderEngine.startProcessing();
+
+      uiAccess.toolbarMain.changeButtonState(
+        ToolbarMainButtonNames.AI_STYLIZE,
+        { disabled: false },
+      );
+    });
     uiEvents.toolbarMain.AI_STYLIZE.onClick(async (event) => {
       uiAccess.toolbarMain.changeButtonState(
         ToolbarMainButtonNames.AI_STYLIZE,
@@ -77,7 +84,7 @@ export class Engine {
         "SLEEP",
         `${sleepytstart.getMinutes()}:${sleepytstart.getSeconds()}`,
       );
-      await this.renderEngine.startProcessing();
+
       const sleeptend = new Date();
       console.log(
         "DONE",
@@ -102,54 +109,7 @@ export class Engine {
   }
 
   // Sandbox is quickly a way to test your idea.
-  public sandbox() {
-    // How to use:
-    // const exampleWorkFunction: WorkFunction<number, number> = async (
-    //   data: number,
-    //   reportProgress: (progress: number) => void,
-    // ) => {
-    //   // Simulate some asynchronous work with progress reporting
-    //   for (let i = 0; i <= 100; i += 20) {
-    //     await new Promise((resolve) => setTimeout(resolve, 200));
-    //     reportProgress(i);
-    //   }
-    //   return data * 2;
-    // };
-
-    // const progressCallback = (progressData: ProgressData) => {
-    //   console.log(
-    //     `Job ${progressData.jobId} progress: ${progressData.progress}%`,
-    //   );
-    // };
-
-    // const workQueue = new WorkQueue<number, number>(
-    //   exampleWorkFunction,
-    //   progressCallback,
-    // );
-    // workQueue.addWork(1, 10);
-    // workQueue.addWork(2, 20);
-    // workQueue.addWork(3, 30);
-
-    // const sharedWorker = new SharedWorker(
-    //   "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts",
-    //   {
-    //     type: "module",
-    //   },
-    // );
-
-    // // Get the port for communication
-    // let port = sharedWorker.port;
-    // port.start();
-    // // Set up the message event listener
-    // port.onmessage = this.onMessage.bind(this);
-    // port.postMessage({ jobID: 1, data: 2, isDoneStreaming: false });
-
-    const value = new SharedWorkerClient<number, number, number>(
-      "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts",
-      undefined,
-    );
-    value.send({ jobID: 1, data: 1, isDoneStreaming: false });
-  }
+  public sandbox() {}
 
   public onMessage(event: MessageEvent) {
     console.log("Message From Shared Worker");

src/KonvaApp/RenderingPrimitives/RenderEngine.ts

@@ -1,7 +1,18 @@
 import { VideoNode } from "../Nodes/VideoNode";
 import Konva from "konva";
+import { RenderTask } from "./RenderTask";
 
-import { RenderTask, WorkerEvent } from "./RenderTask";
+import { DiffusionSharedWorkerClient } from "../SharedWorkers/Diffusion/DiffusionSharedWorkerClient";
+import {
+  SharedWorkerRequest,
+  SharedWorkerResponse,
+} from "../WorkerPrimitives/SharedWorkerBase";
+import {
+  DiffusionSharedWorkerProgressData,
+  DiffusionSharedWorkerResponseData,
+  DiffusionSharedWorker,
+  DiffusionSharedWorkerItemData,
+} from "../SharedWorkers/Diffusion/DiffusionSharedWorker";
 
 // https://www.aiseesoft.com/resource/phone-aspect-ratio-screen-resolution.html#:~:text=16%3A9%20Aspect%20Ratio
 export class RenderEngine {
@@ -9,7 +20,6 @@ export class RenderEngine {
   private offScreenCanvas: OffscreenCanvas;
   private context: OffscreenCanvasRenderingContext2D | null;
 
-  private offscreenCanvasCache: OffscreenCanvas[] = [];
   private isProcessing: boolean;
 
   private frames: ImageBitmap[];
@@ -23,15 +33,23 @@ export class RenderEngine {
   private positionY: number;
 
   private canUseSharedWorker: boolean;
-  private sharedWorker: SharedWorker | undefined;
+
   private port: MessagePort | undefined;
   private maxFrames: number;
+
+  private diffusionWorker:
+    | DiffusionSharedWorkerClient<
+        DiffusionSharedWorkerItemData,
+        DiffusionSharedWorkerResponseData,
+        DiffusionSharedWorkerProgressData
+      >
+    | undefined;
+
   constructor(videoLayer: Konva.Layer, offScreenCanvas: OffscreenCanvas) {
     this.videoNodes = [];
-
     this.isProcessing = false;
 
-    // TODO make this dynamic
+    // TODO: Make this dynamic and update this on change of canvas.
 
     this.width = 720;
     this.height = 1280;
@@ -61,42 +79,29 @@ export class RenderEngine {
 
     this.maxFrames = 7 * 24;
 
-    // DEBUG ONLY
-    // const rectangle = new Konva.Rect({
-    //   x: this.positionX,
-    //   y: this.positionY,
-    //   width: 100,
-    //   height: 100,
-    //   fill: "green",
-    //   stroke: "black",
-    //   strokeWidth: 1,
-    //   draggable: false,
-    // });
-    //this.videoLayer.add(rectangle);
     this.videoLayer.add(captureCanvas);
-    // set back
+    // send back
     captureCanvas.setZIndex(0);
 
-    this.sharedWorker = undefined;
     this.canUseSharedWorker = false;
     this.setupSharedWorker();
+
+    //this.debug();
   }
 
-  // Manage the worker
-  onMessage(event: MessageEvent) {
-    const message = event.data;
-    console.log("Main script received:", message);
-    // Send a response back to the worker
-    if (!this.port) {
-      return console.log("Undefined Worker");
-    }
-  }
-
-  sendMessage() {
-    if (!this.port) {
-      return console.log("Undefined Worker");
-    }
-    this.port.postMessage("Hello from Main Script");
+  debug() {
+    // DEBUG ONLY
+    const rectangle = new Konva.Rect({
+      x: this.positionX,
+      y: this.positionY,
+      width: 100,
+      height: 100,
+      fill: "green",
+      stroke: "black",
+      strokeWidth: 1,
+      draggable: false,
+    });
+    this.videoLayer.add(rectangle);
   }
 
   async sendCanvasPayload(renderTask: RenderTask) {
@@ -106,31 +111,24 @@ export class RenderEngine {
     this.port.postMessage(renderTask);
   }
 
+  onMessageReceived(
+    response: SharedWorkerResponse<
+      DiffusionSharedWorkerResponseData,
+      DiffusionSharedWorkerProgressData
+    >,
+  ) {
+    console.log(response);
+  }
+
   setupSharedWorker() {
     if (typeof SharedWorker !== "undefined") {
-      this.offscreenCanvasCache = [];
-      // canvas size should be defined here
-      for (let i = 0; i < this.maxFrames; i++) {
-        let offScreenCanvas = new OffscreenCanvas(this.width, this.height);
-        this.offscreenCanvasCache.push(offScreenCanvas);
-      }
-
       console.log("Shared Workers are supported in this browser.");
       // Debug chrome://inspect/#workers
-      // TODO ensure that the cors is solved for this.
-      this.sharedWorker = new SharedWorker(
-        "src\\KonvaApp\\RenderingPrimitives\\RenderSharedWorker.ts",
-        {
-          type: "module",
-        },
+      this.diffusionWorker = new DiffusionSharedWorkerClient(
+        "src\\KonvaApp\\SharedWorkers\\Diffusion\\DiffusionSharedWorker.ts",
+        this.onMessageReceived.bind(this),
       );
 
-      // Get the port for communication
-      this.port = this.sharedWorker.port;
-      this.port.start();
-      // Set up the message event listener
-      this.port.onmessage = this.onMessage.bind(this);
-
       this.canUseSharedWorker = true;
     } else {
       console.log("Shared Workers are not supported in this browser.");
@@ -253,7 +251,6 @@ export class RenderEngine {
 
     for (let j = 0; j < largestNumberOfFrames; j++) {
       // Seek Video Nodes first then draw
-
       let frameTime = undefined;
 
       for (let i = 0; i < this.videoNodes.length; i++) {
@@ -300,10 +297,19 @@ export class RenderEngine {
         } // end of for each frame
       } else {
         console.log("Using Shared Worker");
-        const canvas = this.offscreenCanvasCache[j];
-        const context = canvas.getContext("2d");
+
         // decode on the shared webworker.
-        context?.drawImage(
+        if (!this.context) {
+          console.log("Context Didn't Initialize");
+          return;
+        }
+
+        if (!this.diffusionWorker) {
+          console.log("Didnt Initialize Diffusion");
+          return;
+        }
+
+        this.context.drawImage(
           this.videoLayer.canvas._canvas,
           this.positionX,
           this.positionY,
@@ -315,24 +321,26 @@ export class RenderEngine {
           this.height,
         );
 
-        this.sendCanvasPayload({
-          jobID: 1,
-          isDone: false,
-          event: WorkerEvent.DATA,
-          data: {
-            offscreenCanvas: canvas,
-            id: j,
-          },
-        });
+        const data: DiffusionSharedWorkerItemData = {
+          height: this.height,
+          width: this.width,
+          imageBitmap: this.offScreenCanvas.transferToImageBitmap(),
+          frame: j,
+          totalFrames: this.maxFrames,
+        };
+
+        let isDoneStreaming = false;
 
         if (j == this.maxFrames - 1) {
-          this.sendCanvasPayload({
-            jobID: 1,
-            isDone: true,
-            event: WorkerEvent.DATA,
-            data: undefined,
-          });
+          isDoneStreaming = true;
         }
+
+        const container: SharedWorkerRequest<DiffusionSharedWorkerItemData> = {
+          data: data,
+          isDoneStreaming: isDoneStreaming,
+          jobID: 1,
+        };
+        this.diffusionWorker.send(container);
       }
     }
   }

src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorker.ts

@@ -1,4 +1,7 @@
 // Example of this working.
+import { error } from "@techstark/opencv-js";
+import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
+
 import {
   ProgressData,
   WorkResult,
@@ -16,11 +19,16 @@ export interface DiffusionSharedWorkerProgressData {
 }
 
 export interface DiffusionSharedWorkerItemData {
-  canvas: OffscreenCanvas;
+  imageBitmap: ImageBitmap;
+  totalFrames: number;
+  frame: number;
+  height: number;
+  width: number;
 }
 
 export interface DiffusionSharedWorkerResponseData {
   videoUrl: string;
+  zipBlob: Blob; // debug purposes
 }
 
 export class DiffusionSharedWorker extends SharedWorkerBase<
@@ -28,9 +36,29 @@ export class DiffusionSharedWorker extends SharedWorkerBase<
   DiffusionSharedWorkerResponseData,
   DiffusionSharedWorkerProgressData
 > {
+  public zipFileWriter: BlobWriter;
+  public zipWriter: ZipWriter<Blob>;
+  public imageType: string;
+  public totalFrames: number;
+
+  public offscreenCanvas: OffscreenCanvas | undefined;
+  public bitmapContext: ImageBitmapRenderingContext | undefined | null;
+
   constructor(port: MessagePort) {
     super(port);
     this.setup(this.workFunction.bind(this), this.progressFunction.bind(this));
+    this.offscreenCanvas = undefined;
+    this.bitmapContext = undefined;
+    this.imageType = "image/jpeg";
+    this.zipFileWriter = new BlobWriter(this.imageType);
+    this.zipWriter = new ZipWriter(this.zipFileWriter);
+    this.totalFrames = 0;
+  }
+
+  async reset() {
+    this.zipFileWriter = new BlobWriter(this.imageType);
+    this.zipWriter = new ZipWriter(this.zipFileWriter);
+    this.totalFrames = 0;
   }
   // Data here will be shipped off for progressive loading
   async workFunction(
@@ -41,19 +69,41 @@ export class DiffusionSharedWorker extends SharedWorkerBase<
       data: DiffusionSharedWorkerProgressData,
     ) => void,
   ): Promise<[DiffusionSharedWorkerResponseData | undefined, boolean]> {
-    console.log(`Working Item ${item}`);
+    if (this.offscreenCanvas === undefined) {
+      this.offscreenCanvas = new OffscreenCanvas(item.width, item.height);
+      this.bitmapContext = this.offscreenCanvas.getContext("bitmaprenderer");
+    }
 
+    if (!this.bitmapContext) {
+      console.log("Failed to create bitmap context.");
+      throw Error("Bitmap Rendering Context Not Availible.");
+    }
+
+    this.bitmapContext.transferFromImageBitmap(item.imageBitmap);
+
+    const blob = await this.offscreenCanvas.convertToBlob({
+      quality: 1.0,
+      type: this.imageType,
+    });
+
+    const aproxSteps = item.totalFrames * 2;
     const progressData: DiffusionSharedWorkerProgressData = { url: "" };
-    reportProgress(0, progressData);
 
-    // check if stream is done.
+    reportProgress(item.frame / aproxSteps, progressData);
+
+    const name = String(item.frame).padStart(5, "0"); // '0009'
+    await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob));
 
     if (isDoneStreaming === false) {
       return [undefined, false];
     }
 
     // make request via api with options
-    const responseData: DiffusionSharedWorkerResponseData = { videoUrl: "" };
+    const zipBlob = await this.zipWriter.close();
+    const responseData: DiffusionSharedWorkerResponseData = {
+      videoUrl: "",
+      zipBlob: zipBlob,
+    };
 
     return [responseData, true];
   }

src/KonvaApp/SharedWorkers/Diffusion/DiffusionSharedWorkerClient.ts

@@ -1,5 +1,8 @@
-import { SharedWorkerRequest, SharedWorkerResponse } from "./SharedWorkerBase";
-import { ResponseType } from "./SharedWorkerBase";
+import {
+  SharedWorkerRequest,
+  SharedWorkerResponse,
+} from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
+import { ResponseType } from "~/KonvaApp/WorkerPrimitives/SharedWorkerBase";
 
 import {
   DiffusionSharedWorkerItemData,
@@ -29,7 +32,7 @@ export class DiffusionSharedWorkerClient<
       >,
     ) => void,
   ) {
-    // exampl
+    // example
     // "src\\KonvaApp\\WorkerPrimitives\\NumberSharedWorker.ts"
     this.sharedWorker = new SharedWorker(workerPath, {
       type: "module",
@@ -47,13 +50,13 @@ export class DiffusionSharedWorkerClient<
       DiffusionSharedWorkerItemData
     >;
     if (data.responseType === ResponseType.error) {
-      console.log(`TemplateSharedWorkerClient Error:${data}`);
+      console.log(`DiffusionSharedWorkerClient Error:${data}`);
     } else if (data.responseType === ResponseType.progress) {
-      console.log(`TemplateSharedWorkerClient Progress:${data}`);
+      console.log(`DiffusionSharedWorkerClient Progress:${data}`);
     } else if (data.responseType === ResponseType.result) {
-      console.log(`TemplateSharedWorkerClient Result:${data}`);
+      console.log(`DiffusionSharedWorkerClient Result:${data}`);
     } else {
-      console.log("TemplateSharedWorkerClient Message Unknown");
+      console.log(`DiffusionSharedWorkerClient Message Unknown ${data}`);
     }
     console.log(`TemplateSharedWorker Response: ${JSON.stringify(event.data)}`);
   }

src/KonvaApp/index.ts

@@ -4,8 +4,8 @@ export { Engine as EngineType };
 export const KonvaApp = (element: HTMLDivElement) => {
   const engine = new Engine(element);
 
-  engine.sandbox();
-  //engine.initializeStage("");
+  //engine.sandbox();
+  engine.initializeStage("");
 
   return engine;
 };