Feature-Streaming Frames and FFMPEG (#671)

Removed FF MPEG
Added New Rendering code using web workers and zipping and uploading.
Added a New endpoint to upload a bucket of frames.

.env

@@ -8,3 +8,4 @@ REACT_APP_PUBLIC_POSTHOG_KEY="phc_jBFgac0mVALAFk3negnSfYCcHgvkT00yBLQDmCDYNBb"
 REACT_APP_PUBLIC_POSTHOG_UI="https://us.i.posthog.com"
 DEPLOY_CONTEXT="DEVELOPMENT"
 CONTEXT="DEVELOPMENT"
+UPLOAD_API_VIDEO="https://upload.storyteller.ai"

app/Classes/ApiManager/ApiManager.ts

@@ -13,12 +13,14 @@ export class ApiManager {
   ApiTargets: Record<string, string> = {};
 
   constructor() {
+    // look at the .env file
     this.ApiTargets = {
       BaseApi: environmentVariables.values.BASE_API as string,
       GoggleApi: environmentVariables.values.GOOGLE_API as string,
       FunnelApi: environmentVariables.values.FUNNEL_API as string,
       CdnApi: environmentVariables.values.CDN_API as string,
       GravatarApi: environmentVariables.values.GRAVATAR_API as string,
+      uploadApi: environmentVariables.values.UPLOAD_API_VIDEO as string,
     };
   }
 
@@ -152,6 +154,7 @@ export class ApiManager {
       formData.append("file", blob);
     }
 
+    console.log(`HELLO${formData}`);
     return this.fetchMultipartFormData<T>(endpoint, {
       method: "POST",
       headers: {

app/Classes/ApiManager/MediaUploadApi.ts

@@ -2,6 +2,105 @@ import { ApiManager, ApiResponse } from "./ApiManager";
 import { FilterEngineCategories, Visibility } from "~/enums";
 
 export class MediaUploadApi extends ApiManager {
+  private sessionToken: string = undefined;
+
+  private async getSessionTokenForUploadStudioShot() {
+    const endpoint = `${this.ApiTargets.BaseApi}/v1/session_token`;
+    return await this.get<{
+      maybe_signed_session: string;
+    }>({ endpoint })
+      .then((response) => {
+        return {
+          success: true,
+          data: response["maybe_signed_session"],
+        };
+      })
+      .catch((err) => {
+        return {
+          success: false,
+          errorMessage: err.message,
+        };
+      });
+  }
+
+  public async UploadStudioShot({
+    maybe_title,
+    uuid_idempotency_token,
+    blob,
+    fileName,
+    maybe_visibility,
+  }: {
+    maybe_title?: string;
+    uuid_idempotency_token: string;
+    blob: Blob | null;
+    fileName: string;
+    maybe_visibility?: Visibility;
+  }): Promise<ApiResponse<string>> {
+    // This function is a bit special it requires that
+    const endpoint = `${this.ApiTargets.uploadApi}/v1/media_files/upload/studio_shot`;
+
+    if (!this.sessionToken) {
+      console.log("Session Token");
+      const response = await this.getSessionTokenForUploadStudioShot();
+      if (response.success && response.data) {
+        this.sessionToken = response.data;
+        console.log(`Token ${this.sessionToken}`);
+      } else {
+        return {
+          success: false,
+          errorMessage: "Could not obtain session tokens.",
+        };
+      }
+    } else {
+      console.log(`Session Token: ${this.sessionToken}`);
+    }
+
+    const options: Record<string, string | number | undefined> = {
+      maybe_title,
+      maybe_frame_rate_fps: 60,
+      maybe_visibility: maybe_visibility?.toString(),
+    };
+
+    const formRecord = Object.entries(options).reduce(
+      (allOptions, [key, value]) => {
+        if (value === undefined) {
+          return allOptions;
+        }
+        return { ...allOptions, [key]: value.toString() };
+      },
+      {} as Record<string, string>,
+    );
+
+    const formData = new FormData();
+    formData.append("uuid_idempotency_token", uuid_idempotency_token);
+    Object.entries(formRecord).forEach(([key, value]) => {
+      formData.append(key, value);
+    });
+    if (blob && fileName) {
+      formData.append("file", blob, fileName);
+    } else if (blob) {
+      formData.append("file", blob);
+    }
+
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        Accept: "application/json",
+        Session: this.sessionToken,
+      },
+      credentials: "include",
+      body: formData,
+    });
+
+    const result = await response.json();
+
+    if (result.success && result.media_file_token) {
+      return { success: true, data: result.media_file_token };
+    } else {
+      return { success: false, errorMessage: "Failed to Get Media Token" };
+    }
+  }
+
   private async Upload({
     endpoint,
     uuid,

app/api/BucketConfig.ts

@@ -11,6 +11,7 @@ class BucketConfig {
   }
 
   // TODO: Prevent callers with undefined/null paths
+  // TODO pipeline works make more robust....
   getGcsUrl(bucketRelativePath: string | undefined | null): string {
     const bucket = this.getBucket();
     let path = bucketRelativePath;

app/pages/PageEnigma/Editor/VideoProcessor/engine_buffer.ts

@@ -0,0 +1,261 @@
+import { ImageFormat } from "./video_audio_preprocessor";
+import * as THREE from "three";
+import { FrameWorkerManager } from "./workers/frame_worker_manager";
+
+import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
+
+export interface IEngineFrameBuffers {
+  colorFrames: (Blob | null)[];
+  clearBuffer(name: BufferType): Promise<void>;
+  checkIfBufferIsEmpty(buffer: BufferType): Promise<boolean>;
+  countNonNullFrames(buffer: BufferType): Promise<number>;
+}
+
+export enum BufferType {
+  COLOR = "colorFrames",
+}
+
+export interface EngineFrameBuffersError {
+  message: string;
+  code: number;
+}
+
+export class EngineFrameBuffers implements IEngineFrameBuffers {
+  colorFrames: (Blob | null)[];
+
+  private numberOfFrames: number;
+  public hasCache: boolean;
+  public colorRenderQueue: Promise<Blob>[];
+  private webWorkers: boolean;
+
+  public payloadZip: Blob | null;
+
+  public frameWorkerManager: FrameWorkerManager | undefined;
+
+  // for the client side zip.
+  private zipFileWriter: BlobWriter | undefined;
+  private zipWriter: ZipWriter<Blob> | undefined;
+
+  private imageType = "image/jpeg";
+  constructor(width: number, height: number) {
+    this.colorFrames = [];
+    this.colorRenderQueue = [];
+    this.payloadZip = null;
+    this.hasCache = false;
+
+    this.zipFileWriter = undefined;
+    this.zipWriter = undefined;
+
+    //Check for SharedWorker support
+    if (typeof Worker === "undefined") {
+      console.error("Web Workers is not supported in this browser.");
+      this.webWorkers = false;
+    } else {
+      this.webWorkers = true;
+      console.error("Web Workers is supported in this browser.");
+      this.frameWorkerManager = new FrameWorkerManager(
+        width,
+        height,
+        "image/png",
+      );
+    }
+
+    this.webWorkers = false;
+
+    this.imageType = "image/jpeg";
+
+    if (this.webWorkers == false) {
+      this.zipFileWriter = new BlobWriter(this.imageType);
+      this.zipWriter = new ZipWriter(this.zipFileWriter);
+    }
+
+    this.numberOfFrames = 0;
+  }
+
+  async retrieveFrame(
+    rawRenderer: THREE.WebGLRenderer,
+    imageFormat: ImageFormat,
+  ): Promise<Blob> {
+    return new Promise((resolve, reject) => {
+      rawRenderer.domElement.toBlob(
+        (blob) => {
+          if (blob) {
+            const reader = new FileReader();
+            reader.onloadend = () => {
+              //const imgData = reader.result as string;
+              // if needed as a base 64
+              resolve(blob);
+            };
+            reader.onerror = () => {
+              reject({
+                message: "Failed to Retrieve Frame and Serialize it.",
+                code: 100,
+              });
+            };
+            reader.readAsDataURL(blob);
+          }
+        },
+        imageFormat,
+        1.0,
+      );
+    });
+  }
+
+  async setRenderSurfaceSize(width: number, height: number) {
+    if (this.webWorkers && this.frameWorkerManager) {
+      this.frameWorkerManager.setCanvas(width, height);
+    }
+  }
+
+  async enqueueWork(
+    rawRenderer: THREE.WebGLRenderer,
+    scene: THREE.Scene,
+    camera: THREE.PerspectiveCamera,
+  ): Promise<void> {
+    this.numberOfFrames += 1;
+    if (this.webWorkers && this.frameWorkerManager) {
+      this.frameWorkerManager.render(scene, camera, this.numberOfFrames);
+    } else {
+      const colorRenderTask = this.retrieveFrame(rawRenderer, ImageFormat.JPEG);
+      this.colorRenderQueue.push(colorRenderTask);
+    }
+  }
+
+  private downloadBlob(blob: Blob, filename: string): void {
+    // Create an object URL for the Blob
+    const url = URL.createObjectURL(blob);
+
+    // Create a temporary anchor element
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = filename; // Specify the filename
+
+    // Append the anchor to the document body
+    document.body.appendChild(a);
+
+    // Trigger a click event on the anchor
+    a.click();
+
+    // Clean up by revoking the object URL and removing the anchor element
+    URL.revokeObjectURL(url);
+    document.body.removeChild(a);
+  }
+
+  async equeueAudio(audio: Blob) {}
+
+  async collectColorFrames(final_length: number): Promise<void> {
+    try {
+      // webworker requires the COEP.
+      if (this.webWorkers) {
+        console.time("ColorQueueRenderTime Worker Time");
+        if (this.webWorkers && this.frameWorkerManager) {
+          console.log(`Collected Number of Frames: ${final_length}`);
+          // this is sent off to video generation ...
+          this.payloadZip =
+            await this.frameWorkerManager.collectedFrames(final_length);
+        }
+        console.timeEnd("ColorQueueRenderTime Worker Time");
+      } else {
+        console.time("ColorQueueRenderTime");
+        try {
+          // going to have to zip here with the audio
+          if (this.webWorkers == false) {
+            console.log("Using Other Pathway.");
+            this.colorFrames = await Promise.all(this.colorRenderQueue);
+            console.log(this.colorFrames);
+            // Check if zipFileWriter or zipWriter is undefined or null
+            if (!this.zipFileWriter || !this.zipWriter) {
+              console.log("Failed to Startup Zip File Writer");
+              return;
+            }
+
+            this.colorFrames.forEach(async (blob, idx) => {
+              try {
+                if (!blob || !this.zipFileWriter) {
+                  console.log("Failed to get Blob and Zip File Writer");
+                  return;
+                }
+
+                //console.log(`Zipping Frame ${idx} to Zip`);
+                const name = String(idx).padStart(5, "0"); // '0009'
+                if (this.zipWriter) {
+                  await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob));
+                } else {
+                  console.error("zipWriter is undefined");
+                }
+              } catch (error) {
+                console.log(error);
+              }
+            });
+
+            this.payloadZip = await this.zipWriter.close();
+            //this.downloadBlob(this.payloadZip, "test.zip");
+          }
+        } catch (error) {
+          console.log("Error in Collecting Color Frames", error);
+        }
+        console.timeEnd("ColorQueueRenderTime");
+      }
+    } catch (error) {
+      console.log(error);
+    }
+
+    this.hasCache = false;
+  }
+
+  // Testing function
+  public downloadImageBlob(blob: Blob, filename: string) {
+    // Create a URL for the blob
+    const url = URL.createObjectURL(blob);
+
+    // Create a temporary anchor element
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = filename;
+
+    // Append the anchor to the document body
+    document.body.appendChild(a);
+
+    // Trigger a click event on the anchor
+    a.click();
+
+    // Remove the anchor from the document
+    document.body.removeChild(a);
+
+    // Revoke the object URL to free up memory
+    URL.revokeObjectURL(url);
+  }
+
+  async clearBuffer(name: BufferType): Promise<void> {
+    switch (name) {
+      case BufferType.COLOR:
+        this.colorFrames = [];
+        this.colorRenderQueue = [];
+        this.payloadZip = null;
+        this.numberOfFrames = 0;
+
+        // reset the writers
+        this.zipFileWriter = new BlobWriter(this.imageType);
+        this.zipWriter = new ZipWriter(this.zipFileWriter);
+
+        break;
+      default:
+        throw new Error("Invalid buffer name");
+    }
+  }
+
+  async countNonNullFrames(buffer: BufferType): Promise<number> {
+    return this[buffer].filter((frame) => frame !== null).length;
+  }
+
+  async checkIfBufferIsEmpty(buffer: BufferType): Promise<boolean> {
+    return this[buffer].every((frame) => frame === null);
+  }
+
+  async logBufferInfo(buffer: BufferType): Promise<void> {
+    const nonNullFrameCount = await this.countNonNullFrames(buffer);
+    console.log(
+      `Buffer Info - ${buffer}: ${nonNullFrameCount} non-null frames`,
+    );
+  }
+}

app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker.ts

@@ -0,0 +1,15 @@
+// import { VideoAudioPreProcessor } from "app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts";
+
+// const processor = new VideoAudioPreProcessor();
+
+// // chrome://inspect/#workers debug shared worker.
+// self.addEventListener("connect", function (event: MessageEvent) {
+//   const port = (event as MessageEvent & { ports: MessagePort[] }).ports[0];
+
+//   port.addEventListener("message", function (event: MessageEvent) {
+//     console.log("Message received from main script:", event.data);
+//     port.postMessage("Hello from the Shared Worker!");
+//   });
+
+//   port.start();
+// });

app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts

@@ -0,0 +1,31 @@
+class VideoProcessingSharedWorkerManager {
+  constructor() {
+    if (typeof SharedWorker !== "undefined") {
+      console.log("Shared Workers are supported.");
+      // You can safely create a Shared Worker here
+      const worker = new SharedWorker(
+        "app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts",
+        { type: "module" },
+      );
+
+      // Get the port for communication
+      const port = worker.port;
+
+      // Listen for messages from the worker
+      port.addEventListener("message", function (event) {
+        console.log("Message received from worker:", event.data);
+      });
+
+      // Start the port
+      port.start();
+
+      // Send a message to the worker
+      port.postMessage("Hello from the main script!");
+    } else {
+      console.log("Shared Workers are not supported in this browser.");
+    }
+  }
+  async enqueueWork(frames: [string]) {}
+
+  async completedWork() {}
+}

app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts

@@ -1,8 +1,7 @@
 import { MediaFilesApi } from "~/Classes/ApiManager/MediaFilesApi";
 import { BucketConfig } from "~/api/BucketConfig";
-import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
-import { ClipUI } from "../datastructures/clips/clip_ui";
-import * as THREE from "three";
+// import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
+import { ClipUI } from "../../datastructures/clips/clip_ui";
 
 export interface VideoPreProcessorError {
   message: string;
@@ -26,13 +25,6 @@ interface VideoPreProcessorOutputProperties {
   timelineTrackLengthSeconds: number;
 }
 
-export interface EngineFrameBuffers {
-  colorFrames: (string | null)[];
-  normalFrames: (string | null)[];
-  outlineFrames: (string | null)[];
-  depthFrames: (string | null)[];
-}
-
 export interface AudioBuffer {
   buffer: AudioClipInfo[];
 }
@@ -51,104 +43,66 @@ export enum ImageFormat {
 
 // 44099.10498046875 for the image buffer loop and memory pressure.
 export class VideoAudioPreProcessor {
-  ffmpeg: FFmpeg;
+  // ffmpeg: FFmpeg;
   properties: VideoPreProcessorOutputProperties;
   onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void;
   mediaFilesApi: MediaFilesApi;
+
   bucketConfig: BucketConfig;
 
   constructor(
     onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void,
     properties: VideoPreProcessorOutputProperties = {
       frameRate: 60,
-      imageFormat: ImageFormat.JPEG,
+      imageFormat: ImageFormat.PNG,
       timelineTrackLengthSeconds: 7,
     },
   ) {
-    this.ffmpeg = createFFmpeg({ log: false });
+    // this.ffmpeg = createFFmpeg({ log: true, logger: (p) => console.log(p) });
     this.properties = properties;
     this.onProgress = onProgress;
     this.mediaFilesApi = new MediaFilesApi();
+
     this.bucketConfig = new BucketConfig();
   }
 
   // You have to call this early on because it can take a while to load from the CDN.
   async initialize() {
-    if (this.ffmpeg.isLoaded() == false) {
-      await this.ffmpeg.load();
-      console.log("Loaded FFMPEG");
-    } else {
-      console.log("Error Failed to Load FFMPEG");
-    }
+    // if (this.ffmpeg.isLoaded() == false) {
+    //   await this.ffmpeg.load();
+    //   console.log("Video Generation: Loaded FFMPEG");
+    // } else {
+    //   console.log("Video Generation: Already Loaded");
+    // }
   }
 
   // This retrieves by creating dom elements
-  async retrieveFrame(rawRenderer: THREE.WebGLRenderer): Promise<string> {
-    return new Promise((resolve, reject) => {
-      rawRenderer.domElement.toBlob(
-        (blob) => {
-          if (blob) {
-            const reader = new FileReader();
-            reader.onloadend = () => {
-              const imgData = reader.result as string;
-              resolve(imgData);
-            };
-            reader.onerror = () => {
-              reject({
-                message: "Failed to Retrieve Frame and Serialize it.",
-                code: 100,
-              });
-            };
-            reader.readAsDataURL(blob);
-          }
-        },
-        this.properties.imageFormat,
-        1.0,
-      );
-    });
+
+  async removeInMemoryWavs(): Promise<void> {
+    try {
+      // const fileNames = this.ffmpeg.FS("readdir", "/");
+      // const extension = ".wav";
+      // for (const fileName of fileNames) {
+      //   if (fileName.endsWith(extension)) {
+      //     await this.ffmpeg.FS("unlink", fileName);
+      //   }
+      // }
+    } catch (error) {
+      const processError: VideoPreProcessorError = {
+        message: "Failed to delete files from FFmpeg memory store:",
+        code: 101,
+      };
+
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      console.error(errorMessage);
+
+      throw processError;
+    }
   }
-  // parallel version
-  // async retrieveFrame(rawRenderer: THREE.WebGLRenderer): Promise<string> {
-  //   return new Promise((resolve, reject) => {
-  //     // Create an offscreen canvas
-  //     const offscreenCanvas = document.createElement("canvas");
-  //     offscreenCanvas.width = rawRenderer.domElement.width;
-  //     offscreenCanvas.height = rawRenderer.domElement.height;
-  //     const ctx = offscreenCanvas.getContext("2d");
-
-  //     if (!ctx) {
-  //       reject("Failed to get 2D context");
-  //       return;
-  //     }
-
-  //     // Draw the renderer's canvas content onto the offscreen canvas
-  //     ctx.drawImage(rawRenderer.domElement, 0, 0);
-
-  //     // Convert the offscreen canvas to a Blob
-  //     offscreenCanvas.toBlob(
-  //       (blob) => {
-  //         if (blob) {
-  //           const reader = new FileReader();
-  //           reader.onloadend = () => {
-  //             const imgData = reader.result as string;
-  //             resolve(imgData);
-  //           };
-  //           reader.onerror = () => {
-  //             reject("Failed to Retrieve Frame");
-  //           };
-  //           reader.readAsDataURL(blob);
-  //         } else {
-  //           reject("Failed to convert canvas to Blob");
-  //         }
-  //       },
-  //       "image/jpeg",
-  //       1.0,
-  //     );
-  //   });
-  // }
   async removeInMemoryImages(): Promise<void> {
     try {
-      const fileNames = this.ffmpeg.FS("readdir", "/");
+      // const fileNames = this.ffmpeg.FS("readdir", "/");
       const imageFormat = this.properties.imageFormat;
       let extension = "";
       if (imageFormat == ImageFormat.JPEG) {
@@ -157,11 +111,11 @@ export class VideoAudioPreProcessor {
         extension = ".png";
       }
 
-      for (const fileName of fileNames) {
-        if (fileName.endsWith(extension)) {
-          await this.ffmpeg.FS("unlink", fileName);
-        }
-      }
+      // for (const fileName of fileNames) {
+      //   if (fileName.endsWith(extension)) {
+      //     await this.ffmpeg.FS("unlink", fileName);
+      //   }
+      // }
     } catch (error) {
       const processError: VideoPreProcessorError = {
         message: "Failed to delete files from FFmpeg memory store:",
@@ -269,12 +223,12 @@ export class VideoAudioPreProcessor {
       });
 
       if (response.success && response.data) {
-        this.bucketConfig.isLocalDev = false; // TODO REMOVE
-        const imageUrl = this.bucketConfig.getGcsUrl(
+        this.bucketConfig.isLocalDev = false;
+        const mediaUrl = this.bucketConfig.getGcsUrl(
           response.data.public_bucket_path,
         );
 
-        return imageUrl;
+        return mediaUrl;
       } else {
         const processError: VideoPreProcessorError = {
           message: "Failed to get MediaUrlFrom Token",
@@ -363,86 +317,119 @@ export class VideoAudioPreProcessor {
   ): Promise<Blob | void> {
     const outputName = `${audioOutputFileName}`;
 
-    // Create the samples in memory
-    for (let i = 0; i < audioBuffer.buffer.length; i++) {
-      const buffer: AudioClipInfo = audioBuffer.buffer[i];
-      const urlPath = await this.getMediaUrlFromToken(buffer.media_id);
-      const audio = await fetchFile(urlPath);
+    try {
+      // Create the samples in memory
+      for (let i = 0; i < audioBuffer.buffer.length; i++) {
+        const buffer: AudioClipInfo = audioBuffer.buffer[i];
+        const urlPath = await this.getMediaUrlFromToken(buffer.media_id);
+        const audio = await fetchFile(urlPath);
+        console.log(urlPath);
+        console.log(audio);
 
-      if (audio.length === 0) {
-        console.log("audio is 0");
+        if (audio.length === 0) {
+          console.log("audio is 0");
 
-        const processError: VideoPreProcessorError = {
-          message: `Audio length is 0 for ${urlPath}`,
-          code: 103,
-        };
-        throw processError;
+          const processError: VideoPreProcessorError = {
+            message: `Audio length is 0 for ${urlPath}`,
+            code: 103,
+          };
+          throw processError;
+        }
+
+        await this.ffmpeg.FS("writeFile", `tmp${i}.wav`, audio);
       }
+      console.log("Write the Audio");
 
-      await this.ffmpeg.FS("writeFile", `tmp${i}.wav`, audio);
-    }
-    console.log("Write the Audio");
+      // Construct the FFmpeg command to mix the WAV files
+      const inputArgs = [];
+      const filterComplexArgs = [];
 
-    // Construct the FFmpeg command to mix the WAV files
-    const inputArgs = [];
-    const filterComplexArgs = [];
-
-    // Add the silent audio file and account for it below. i + 1
-    inputArgs.push(
-      "-f",
-      "lavfi",
-      "-t",
-      "7",
-      "-i",
-      "anullsrc=r=44100:cl=stereo",
-    );
-
-    // Add the existing audio clips
-    for (let i = 0; i < audioBuffer.buffer.length; i++) {
-      const endTime = audioBuffer.buffer[i].length / this.properties.frameRate; // allows trimming from the right
-
-      const offset = audioBuffer.buffer[i].offset / this.properties.frameRate; // where it will end up on the track.
-
-      const startTime =
-        audioBuffer.buffer[i].offset / this.properties.frameRate; // allows trimming from the left eventually
-      const end = endTime - startTime;
-
-      inputArgs.push("-i", `tmp${i}.wav`);
-      // trim the audio to the end time then offset
-      filterComplexArgs.push(
-        `[${i + 1}:a]atrim=start=${0}:end=${end},adelay=${offset * 1000}|${offset * 1000}[a${i}];`,
+      // Add the silent audio file and account for it below. i + 1
+      inputArgs.push(
+        "-f",
+        "lavfi",
+        "-t",
+        "7",
+        "-i",
+        "anullsrc=r=44100:cl=stereo",
       );
 
-      console.log("Write Audio and Offset Position");
+      // Add the existing audio clips
+      for (let i = 0; i < audioBuffer.buffer.length; i++) {
+        const endTime =
+          audioBuffer.buffer[i].length / this.properties.frameRate; // allows trimming from the right
+
+        const offset = audioBuffer.buffer[i].offset / this.properties.frameRate; // where it will end up on the track.
+
+        const startTime =
+          audioBuffer.buffer[i].offset / this.properties.frameRate; // allows trimming from the left eventually
+        const end = endTime - startTime;
+
+        inputArgs.push("-i", `tmp${i}.wav`);
+        // trim the audio to the end time then offset
+        filterComplexArgs.push(
+          `[${i + 1}:a]atrim=start=${0}:end=${end},adelay=${offset * 1000}|${offset * 1000}[a${i}];`,
+        );
+
+        console.log("Write Audio and Offset Position");
+      }
+
+      // Add the silent audio track to the filter complex arguments
+      filterComplexArgs.push("[0:a]");
+
+      // Add the existing audio tracks to the filter complex arguments
+      for (let i = 0; i < audioBuffer.buffer.length; i++) {
+        filterComplexArgs.push(`[a${i}]`);
+      }
+
+      // Mix the silent audio track with the existing audio tracks
+      filterComplexArgs.push(`amix=inputs=${audioBuffer.buffer.length + 1}[a]`);
+
+      // Run the FFmpeg command to mix the WAV files
+      await this.ffmpeg.run(
+        ...inputArgs,
+        "-filter_complex",
+        filterComplexArgs.join(""),
+        "-map",
+        "[a]",
+        `${outputName}.wav`,
+      );
+
+      this.debugListFiles();
+
+      if (includeBlob) {
+        const blob = await this.outputWavFromMemory(`${outputName}`);
+        return blob;
+      } else {
+        return;
+      }
+    } catch (err) {
+      console.log(err);
+      throw err;
     }
+  }
 
-    // Add the silent audio track to the filter complex arguments
-    filterComplexArgs.push("[0:a]");
+  // Testing function
+  public downloadImageBlob(blob: Blob, filename: string) {
+    // Create a URL for the blob
+    const url = URL.createObjectURL(blob);
 
-    // Add the existing audio tracks to the filter complex arguments
-    for (let i = 0; i < audioBuffer.buffer.length; i++) {
-      filterComplexArgs.push(`[a${i}]`);
-    }
+    // Create a temporary anchor element
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = filename;
 
-    // Mix the silent audio track with the existing audio tracks
-    filterComplexArgs.push(`amix=inputs=${audioBuffer.buffer.length + 1}[a]`);
+    // Append the anchor to the document body
+    document.body.appendChild(a);
 
-    // Run the FFmpeg command to mix the WAV files
-    await this.ffmpeg.run(
-      ...inputArgs,
-      "-filter_complex",
-      filterComplexArgs.join(""),
-      "-map",
-      "[a]",
-      `${outputName}.wav`,
-    );
+    // Trigger a click event on the anchor
+    a.click();
 
-    if (includeBlob) {
-      const blob = await this.outputWavFromMemory(`${outputName}`);
-      return blob;
-    } else {
-      return;
-    }
+    // Remove the anchor from the document
+    document.body.removeChild(a);
+
+    // Revoke the object URL to free up memory
+    URL.revokeObjectURL(url);
   }
 
   async processVideoWithBuffer(
@@ -460,8 +447,9 @@ export class VideoAudioPreProcessor {
       } else {
         extension = ".png";
       }
-      // console.log("Image Files");
-      // console.log(`Buffer Lenght ${buffer.length}`);
+      console.log(`Image Files of extension ${extension}`);
+      console.log(`Buffer Length ${buffer.length}`);
+
       for (let index = 0; index < buffer.length; index++) {
         const element = buffer[index];
         if (element === null) {
@@ -469,6 +457,7 @@ export class VideoAudioPreProcessor {
         }
         // this can fail so catch any errors.
         const image = await fetchFile(element);
+
         await this.ffmpeg.FS(
           "writeFile",
           `${imagePrefix}_image${index}${extension}`,
@@ -529,6 +518,18 @@ export class VideoAudioPreProcessor {
     return blob;
   }
 
+  // Grab the file from in memory fs and provide it as a blob
+  async outputImageFromMemory(
+    fileName: string,
+    imageExtension: ImageFormat = ImageFormat.JPEG,
+  ): Promise<Blob> {
+    // Upload individual videos or single color video
+    const output = this.ffmpeg.FS("readFile", `${fileName}`);
+    // Create a Blob from the output file for downloading
+    const blob = new Blob([output.buffer], { type: `${imageExtension}` });
+    return blob;
+  }
+
   async outputWavFromMemory(fileName: string): Promise<Blob> {
     // Upload individual videos or single color video
     const output = this.ffmpeg.FS("readFile", `${fileName}.wav`);
@@ -542,8 +543,6 @@ export class VideoAudioPreProcessor {
     // List all files in the in-memory filesystem
     const files = this.ffmpeg.FS("readdir", "/");
     await this.deleteFilesFromFFmpegMemoryStore(files);
-    // This is undefined probably never want to call this really lol
-    // this.ffmpeg.exit();
   }
 
   async debugListFiles() {

app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts

@@ -0,0 +1,162 @@
+interface Task {
+  imageBitmap: ImageBitmap;
+  width: number;
+  height: number;
+  id: number;
+}
+
+import { ClipUI } from "~/pages/PageEnigma/datastructures/clips/clip_ui";
+import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
+
+class FrameProcessorWorker {
+  private holdingCanvas: OffscreenCanvas;
+  private ctx: ImageBitmapRenderingContext | null;
+  public imageType: string;
+
+  private zipFileWriter: BlobWriter;
+  private zipWriter: ZipWriter<Blob>;
+
+  private decodeQueue: Task[];
+
+  public audio_data: ClipUI[];
+
+  public isProcessing: boolean;
+
+  public final_length: number;
+  public item_count: number;
+  constructor() {
+    this.final_length = 0;
+    this.holdingCanvas = new OffscreenCanvas(0, 0);
+    this.ctx = this.holdingCanvas.getContext("bitmaprenderer");
+    this.imageType = "image/jpeg";
+
+    // Zipping
+    this.decodeQueue = [];
+    this.zipFileWriter = new BlobWriter(this.imageType);
+    this.zipWriter = new ZipWriter(this.zipFileWriter);
+
+    this.audio_data = [];
+
+    this.isProcessing = false;
+    this.item_count = 0;
+  }
+
+  async startZipProcessQueue(): Promise<void> {
+    if (this.decodeQueue.length > 0 && this.isProcessing == false) {
+      this.isProcessing = true;
+
+      const task = this.decodeQueue.shift();
+      if (task) {
+        const { imageBitmap, width, height, id } = task;
+        // Proceed with processing the task
+
+        //console.log(`Working On an Frame Id ${id}`);
+        this.holdingCanvas.width = width;
+        this.holdingCanvas.height = height;
+
+        try {
+          // Create or reuse the offscreen canvas
+          this.ctx?.transferFromImageBitmap(imageBitmap);
+
+          // Convert the canvas content to a Blob
+          const blob = await this.holdingCanvas.convertToBlob({
+            quality: 1.0,
+            type: this.imageType,
+          });
+
+          // Send the Blob back to the main thread
+          //console.log(`Zipping Frame ${id} to Zip`);
+          const name = String(id).padStart(5, "0"); // '0009'
+          await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob));
+
+          this.item_count++;
+        } catch (error) {
+          console.error("Error processing image:", error);
+          self.postMessage({ error: (error as Error).message }); //
+        } finally {
+          this.isProcessing = false;
+        }
+      } else {
+        console.log("Queue is empty, no task to process.");
+        return;
+      }
+    }
+    // have this function loop
+    setTimeout(this.startZipProcessQueue.bind(this), 0);
+  }
+
+  async zipFile() {
+    // ensure that this queue is done before zipping
+    console.log(`Waiting to Finish Decoding: ${this.decodeQueue.length}`);
+    console.log(`Decoded Count: ${this.item_count}`);
+    console.log(`Final Length: ${this.final_length}`);
+    if (this.item_count === this.final_length) {
+      console.log("Zipping");
+      const result = await this.zipWriter.close();
+      self.postMessage({ result: result });
+      // reset the item count and final length
+      this.item_count = 0;
+      this.final_length = 0;
+
+      // Zipping
+      this.decodeQueue = [];
+      this.zipFileWriter = new BlobWriter(this.imageType);
+      this.zipWriter = new ZipWriter(this.zipFileWriter);
+      this.audio_data = [];
+      this.isProcessing = false;
+
+      return;
+    } else {
+      setTimeout(this.zipFile.bind(this), 0);
+    }
+  }
+
+  public enqueueTask(
+    imageBitmap: ImageBitmap,
+    width: number,
+    height: number,
+    id: number,
+  ): void {
+    this.decodeQueue.push({ imageBitmap, width, height, id });
+  }
+}
+
+const runner = new FrameProcessorWorker();
+let start = false;
+
+self.addEventListener("message", async (event: MessageEvent) => {
+  const {
+    type,
+    imageBitmap,
+    width,
+    height,
+    frame,
+    audio_data,
+    zip,
+    final_length,
+  } = event.data;
+
+  // console.log(
+  //   `${type}: ${imageBitmap} ${width} ${height} ${frame} ${audio_data} ${zip} ${final_length}`,
+  // );
+
+  if (start == false) {
+    start = true;
+    runner.imageType = type;
+    await runner.startZipProcessQueue();
+  }
+
+  if (audio_data || zip) {
+    if (audio_data) {
+      runner.audio_data = audio_data;
+    }
+    if (zip) {
+      console.log(`Worker Reported Final Length: ${final_length}`);
+      runner.final_length = final_length;
+      await runner.zipFile();
+    }
+    return;
+  }
+
+  runner.enqueueTask(imageBitmap, width, height, frame);
+});

app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_worker_manager.ts

@@ -0,0 +1,125 @@
+import * as THREE from "three";
+// import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js";
+
+interface WorkerMessageEvent extends MessageEvent {
+  data: {
+    result: Blob;
+  };
+}
+
+export class FrameWorkerManager {
+  private worker: Worker;
+  private offscreenCanvas: OffscreenCanvas;
+  private offScreenRenderer: THREE.WebGLRenderer;
+  public type: string;
+
+  private blobResult: Blob | undefined;
+
+  constructor(width: number, height: number, type: string = "image/jpeg") {
+    this.type = type;
+
+    this.worker = new Worker(
+      "app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts",
+      { type: "module" },
+    );
+    this.worker.addEventListener("message", this.handleMessage.bind(this));
+
+    this.offscreenCanvas = new OffscreenCanvas(width, height);
+    this.offScreenRenderer = new THREE.WebGLRenderer({
+      canvas: this.offscreenCanvas,
+    });
+  }
+
+  setCanvas(width: number, height: number): void {
+    this.offscreenCanvas.width = width;
+    this.offscreenCanvas.height = height;
+  }
+
+  handleMessage(event: WorkerMessageEvent): void {
+    const { result } = event.data;
+    console.log(result);
+    // zip file
+    this.blobResult = result;
+  }
+
+  async sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async collectedFrames(final_length: number): Promise<Blob> {
+    // to get the zip file
+    console.log(`The Last Frame Number is: ${final_length}`);
+    this.worker.postMessage({
+      type: this.type,
+      imageBitmap: undefined,
+      width: undefined,
+      height: undefined,
+      frame: undefined,
+      audio_data: undefined,
+      zip: true,
+      final_length: final_length,
+    });
+
+    while (this.blobResult === undefined) {
+      await this.sleep(1);
+    }
+
+    const result = this.blobResult;
+    this.blobResult = undefined;
+
+    // For testing.
+    //this.downloadBlob(result, "test.zip");
+
+    if (!result) {
+      throw Error("Failed to Compress Payload");
+    }
+    return result;
+  }
+
+  private downloadBlob(blob: Blob, filename: string): void {
+    // Create an object URL for the Blob
+    const url = URL.createObjectURL(blob);
+
+    // Create a temporary anchor element
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = filename; // Specify the filename
+
+    // Append the anchor to the document body
+    document.body.appendChild(a);
+
+    // Trigger a click event on the anchor
+    a.click();
+
+    // Clean up by revoking the object URL and removing the anchor element
+    URL.revokeObjectURL(url);
+    document.body.removeChild(a);
+  }
+
+  public terminate(): void {
+    this.worker.terminate();
+    console.log("Worker Terminated");
+  }
+
+  public render(scene: THREE.Scene, camera: THREE.Camera, frame: number): void {
+    this.offScreenRenderer.render(scene, camera);
+    const imageBitmap = this.offscreenCanvas.transferToImageBitmap();
+    this.processFrame(imageBitmap, frame);
+  }
+
+  private processFrame(imageBitmap: ImageBitmap, frame: number): void {
+    const width = this.offscreenCanvas.width;
+    const height = this.offscreenCanvas.height;
+    const type = this.type;
+
+    this.worker.postMessage({
+      type,
+      imageBitmap,
+      width,
+      height,
+      audio_data: undefined,
+      frame,
+      zip: false,
+    });
+  }
+}

app/pages/PageEnigma/Editor/editor.ts

@@ -36,6 +36,7 @@ import {
   loadingBarIsShowing,
   signalScene,
 } from "~/signals";
+
 import { outlinerState, updateObjectPanel } from "../signals";
 import { IGenerationOptions } from "../models/generationOptions";
 import { toEngineActions } from "../Queue/toEngineActions";
@@ -45,10 +46,11 @@ import { SceneManager } from "./scene_manager_api";
 import { CustomOutlinePass } from "./CustomOutlinePass.js";
 import FindSurfaces from "./FindSurfaces.js";
 import {
-  EngineFrameBuffers,
-  VideoAudioPreProcessor,
-} from "./video_audio_preprocessor";
-import { forEach } from "lodash";
+  ImageFormat,
+  //VideoAudioPreProcessor,
+} from "./VideoProcessor/video_audio_preprocessor";
+import { BufferType, EngineFrameBuffers } from "./VideoProcessor/engine_buffer";
+
 import Stats from "three/examples/jsm/libs/stats.module.js";
 
 export type EditorInitializeConfig = {
@@ -93,7 +95,7 @@ class Editor {
   cameraViewControls: FreeCam | undefined;
   orbitControls: OrbitControls | undefined;
   locked: boolean;
-  frame_buffer: string[][] = []; // Contains a single frame with 4 channels or a signle frame with 1 channel color | depth | outline | normal.
+
   render_timer: number;
   fps_number: number;
   cap_fps: number;
@@ -174,50 +176,29 @@ class Editor {
   surfaceFinder: FindSurfaces | undefined;
 
   // New Rendering Pipeline Engine Work
-  videoAudioPreProcessor: VideoAudioPreProcessor;
-  colorRenderQueue: Array<Promise<string> | null>;
-  depthRenderQueue: Array<Promise<string> | null>;
-  normalRenderQueue: Array<Promise<string> | null>;
-  outlineRenderQueue: Array<Promise<string> | null>;
+  //videoAudioPreProcessor: VideoAudioPreProcessor | undefined;
+
   engineFrameBuffers: EngineFrameBuffers;
   renderIndex: number;
   // this should be set in the future to extend the lenght of the track for rendering engine
   globalSetTrackLengthSeconds: number;
 
+  // this is to prevent recording processing from happening twice there is an update loop bug at its core.
+  processingRecording: boolean;
+
+  // this is to catch and ensure that caching doesn't break the app.
+  // this happens because we can error out during the video generation process and things will cache despite that failing.
+  processingHasFailed: boolean;
   stats: Stats;
 
   constructor() {
+    this.processingHasFailed = false;
     console.log(
       "If you see this message twice! then it rendered twice, if you see it once it's all good.",
     );
-
+    this.can_initialize = true;
+    this.processingRecording = false;
     this.stats = new Stats();
-
-
-    // New Rendering Pipeline Engine Work
-    this.globalSetTrackLengthSeconds = 7;
-    this.cap_fps = 60;
-    const frames = this.globalSetTrackLengthSeconds * this.cap_fps;
-
-    this.videoAudioPreProcessor = new VideoAudioPreProcessor();
-    this.videoAudioPreProcessor.initialize();
-
-    // number of frames max in a scene
-    this.colorRenderQueue = new Array(frames).fill(null);
-    this.depthRenderQueue = new Array(frames).fill(null);
-    this.normalRenderQueue = new Array(frames).fill(null);
-    this.outlineRenderQueue = new Array(frames).fill(null);
-    this.renderIndex = 0;
-
-    this.engineFrameBuffers = {
-      colorFrames: new Array(frames).fill(null),
-      normalFrames: new Array(frames).fill(null),
-      outlineFrames: new Array(frames).fill(null),
-      depthFrames: new Array(frames).fill(null),
-    };
-
-    // New Rendering Pipeline Engine Work
-
     // TODO: REMOVE LATER WITH BETTER FIX FOR IMPORTING AMMOJS
     document.body.appendChild(
       Object.assign(document.createElement("script"), {
@@ -225,8 +206,6 @@ class Editor {
       }),
     );
 
-    this.can_initialize = false;
-    this.can_initialize = true;
     const newElement = document.createElement("div");
     newElement.id = "created-one-element";
     document.body.appendChild(newElement);
@@ -322,6 +301,24 @@ class Editor {
     this.globalIpAdapterImage = undefined; // used to display when loading in the app. and to serialize to an image token
     // TODO REMOVE
     this.outliner_feature_flag = true;
+
+    // New Rendering Pipeline Engine Work
+    this.globalSetTrackLengthSeconds = 7;
+
+    // set image type at this stage
+
+    const imageFormat = ImageFormat.JPEG;
+
+    this.renderIndex = 0;
+    this.engineFrameBuffers = new EngineFrameBuffers(
+      this.getRenderDimensions().width,
+      this.getRenderDimensions().height,
+    );
+    if (this.engineFrameBuffers.frameWorkerManager) {
+      this.engineFrameBuffers.frameWorkerManager.type = imageFormat;
+    }
+
+    // New Rendering Pipeline Engine Work
   }
   getRenderDimensions() {
     switch (this.render_camera_aspect_ratio) {
@@ -413,6 +410,7 @@ class Editor {
       data: this.render_camera_aspect_ratio,
     });
   }
+
   initialize({
     sceneToken,
     editorCanvasEl,
@@ -426,6 +424,9 @@ class Editor {
 
     this.can_initialize = false;
 
+    // This is to prevent recording processing from happening twice there is an update loop bug at its core.
+    this.processingRecording = false;
+
     // Gets the canvas.
     this.canvReference = editorCanvasEl;
     this.canvasRenderCamReference = camViewCanvasEl;
@@ -473,16 +474,10 @@ class Editor {
 
     this.renderer.shadowMap.enabled = true;
     this.clock = new THREE.Clock();
+
     // Resizes the renderer.
     this.renderer.setSize(width, height);
     this.renderer.setPixelRatio(window.devicePixelRatio);
-    //document.body.appendChild(this.renderer.domElement)
-
-    // window.addEventListener("resize", this.onWindowResize.bind(this));
-    // this.renderer.domElement.addEventListener(
-    //   "resize",
-    //   this.onWindowResize.bind(this),
-    // );
 
     this._configurePostProcessing();
     // Controls and movement.
@@ -575,7 +570,7 @@ class Editor {
       this.getAssetType.bind(this),
       this.setSelected.bind(this),
       this.isMovable.bind(this),
-      this.enable_stats.bind(this)
+      this.enable_stats.bind(this),
     );
 
     if (this.outliner_feature_flag) {
@@ -638,7 +633,7 @@ class Editor {
   }
 
   public enable_stats() {
-    document.body.appendChild(this.stats.dom)
+    document.body.appendChild(this.stats.dom);
   }
 
   public async newScene(sceneTitleInput: string) {
@@ -810,16 +805,22 @@ class Editor {
       this.activeScene.scene,
       this.render_camera,
     );
+
     this.render_composer = new EffectComposer(this.rawRenderer, renderTarget);
+
     this.surfaceFinder = new FindSurfaces();
+
     this.rawRenderPass = new RenderPass(
       this.activeScene.scene,
       this.render_camera,
     );
+
     this.render_composer.addPass(this.rawRenderPass);
+
     this.render_composer.addPass(this.customOutlinerPass);
 
     this.renderOutputPass = new OutputPass();
+
     this.render_composer.addPass(this.renderOutputPass);
 
     this.setColorMap();
@@ -883,7 +884,6 @@ class Editor {
     this.composer.addPass(this.outlinePass);
 
     this.saoPass = new SAOPass(this.activeScene.scene, this.camera);
-
     this.saoPass.params.saoBias = 4.1;
     this.saoPass.params.saoIntensity = 1.0;
     this.saoPass.params.saoScale = 32.0;
@@ -955,11 +955,15 @@ class Editor {
               : 1000;
 
         this.rawRenderer.setSize(width, height);
+
+        // ensure that during a resize or change in perspective this will processes correct.
+        this.engineFrameBuffers.setRenderSurfaceSize(width, height);
+
         this.render_camera.aspect = width / height;
       }
 
       this.render_timer += this.clock.getDelta();
-      this.frames += 1;
+
       this.playback_location++;
 
       this.utils.removeTransformControls(true);
@@ -967,53 +971,29 @@ class Editor {
       if (this.timeline.is_playing) {
         this.setColorMap();
         this.render_composer?.render();
+        if (this.canvReference) {
+          console.time("Color Frame RenderTime");
 
-        // TODO we can fix this, we need to create a bunch of off screen canvas elements
-        // then in parallel have them be peeled off of there
-        const renderTask = this.videoAudioPreProcessor.retrieveFrame(
-          this.rawRenderer,
-        );
-        // await for now race condition with frame retrival on the buffer.
-        this.engineFrameBuffers.colorFrames[this.renderIndex] =
-          await renderTask;
+          // reset this flag to ensure that a failure isn't cached
+          this.processingHasFailed = false;
+          // can't render without this.
 
-        if (this.engine_preprocessing) {
-          // Normals
-          this.setNormalMap();
-          this.render_composer?.render();
+          if (!this.render_camera) {
+            return;
+          }
 
-          const normalRenderTask = this.videoAudioPreProcessor.retrieveFrame(
+          this.engineFrameBuffers.enqueueWork(
             this.rawRenderer,
+            this.activeScene.scene,
+            this.render_camera,
           );
-          this.engineFrameBuffers.normalFrames[this.renderIndex] =
-            await normalRenderTask;
-
-          //this.normalRenderQueue[this.renderIndex] = normalRenderTask;
-
-          // Depth
-          this.setRenderDepth();
-          this.render_composer?.render();
-          const depthRenderTask = this.videoAudioPreProcessor.retrieveFrame(
-            this.rawRenderer,
-          );
-          this.engineFrameBuffers.depthFrames[this.renderIndex] =
-            await depthRenderTask;
-          // this.depthRenderQueue[this.renderIndex] = depthRenderTask;
-
-          // Outline
-          this.setOutlineRender();
-          this.render_composer?.render();
-          const outlineRenderTask = this.videoAudioPreProcessor.retrieveFrame(
-            this.rawRenderer,
-          );
-          this.engineFrameBuffers.outlineFrames[this.renderIndex] =
-            await outlineRenderTask;
-
-          //this.outlineRenderQueue[this.renderIndex] = outlineRenderTask;
 
           this.renderIndex += 1;
+          console.log(`Frames Counted: ${this.renderIndex}`);
+
+          console.timeEnd("Color Frame RenderTime");
         } else {
-          this.renderIndex += 1;
+          console.log("We lost the canvas reference.");
         }
 
         this.render_timer += this.clock.getDelta();
@@ -1021,38 +1001,35 @@ class Editor {
       if (!this.timeline.is_playing) {
         this.playback_location = 0;
         try {
-          this.renderIndex = 0;
-          // collect all frames here
-          console.time("RenderQueueTime");
-          //const colors = await Promise.all(this.colorRenderQueue);
-          console.timeEnd("RenderQueueTime");
-          // console.log(`Number of Frames: ${colors.length}`);
-
-          // We are doing it sync now.
-          // this.engineFrameBuffers.colorFrames = colors;
-          this.colorRenderQueue.fill(null);
-
-          if (this.engine_preprocessing) {
-            console.time("EnginePreProcessingTime");
-            //const normals = await Promise.all(this.normalRenderQueue);
-            //this.engineFrameBuffers.normalFrames = normals;
-            //this.normalRenderQueue.fill(null);
-
-            //const depth = await Promise.all(this.depthRenderQueue);
-            //this.engineFrameBuffers.depthFrames = depth;
-            //this.depthRenderQueue.fill(null);
-
-            //const outlines = await Promise.all(this.outlineRenderQueue);
-            //this.engineFrameBuffers.outlineFrames = outlines;
-            //this.outlineRenderQueue.fill(null);
-            console.timeEnd("EnginePreProcessingTime");
+          if (this.processingRecording) {
+            console.log(
+              "processingRecording is already happening this shouldn't have happened",
+            );
+            return;
           }
+          this.processingRecording = true;
+          // collect all frames here, there is some kind of race condition that is happening
+          // TODO BUG with the main loop,that sends multiple zip requests.
+          console.log(`COLLECTING COLOR FRAMES COUNTED: ${this.renderIndex}`);
+          await this.engineFrameBuffers.collectColorFrames(this.renderIndex);
 
+          await this.engineFrameBuffers.logBufferInfo(BufferType.COLOR);
+          this.renderIndex = 0;
+          console.time("Stop Playback And Upload Video Time");
           await this.stopPlaybackAndUploadVideo();
+          console.timeEnd("Stop Playback And Upload Video Time");
+
+          this.processingRecording = false;
+          console.log("Processing has ended");
         } catch (error) {
-          console.log(error);
+          // don't use cache in this case.
+          this.processingHasFailed = true;
+          this.engineFrameBuffers.clearBuffer(BufferType.COLOR);
+          console.log(`Video Generation: ${error}`);
         }
-      }
+      } // End Timeline Playing
+
+      // BRB ^)^
     }
   }
 
@@ -1240,15 +1217,13 @@ class Editor {
   async generateVideo() {
     // cannot run this function reliably without ensuring state below doesn't blow everything up.
     if (await this.checkAndUseCache()) {
-      console.log("Checking Cache");
+      console.log("Generating Video: Checking Cache");
       await this.videoGeneration.handleCachedEnqueue();
       return;
     }
 
     // some state changes below
 
-    console.log("Generating Video", this.frame_buffer);
-
     this.timeline.is_playing = false;
     this.timeline.scrubber_frame_position = 0;
     this.timeline.current_time = 0;
@@ -1266,7 +1241,6 @@ class Editor {
     console.log("Running without Cache");
 
     this.togglePlayback();
-    this.frame_buffer = [];
     this.render_timer = 0;
     this.activeScene.renderMode(this.rendering);
     this.timeline.scrubber_frame_position = 0;
@@ -1283,7 +1257,10 @@ class Editor {
   // This sadly doesn't cover camera framing style changes,
   // Reasoning is that there is no defined interface to get changes like that.
   async checkAndUseCache(): Promise<boolean> {
-    if (await this.useCachedMediaTokens()) {
+    if (
+      (await this.useCachedMediaTokens()) &&
+      this.processingHasFailed == false
+    ) {
       console.log("Using Cache");
       return true;
     } else {
@@ -1425,6 +1402,7 @@ class Editor {
 
   setupResizeObserver() {
     this.containerMayReset();
+
     if (!this.container) {
       return;
     }

app/pages/PageEnigma/Editor/preview_video_generation.ts

@@ -1,16 +1,12 @@
 import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager";
 
 import {
-  StandardResponse,
   ProcessStatus,
   StreamingProgressResponse,
   VideoAudioPreProcessor,
-  EngineFrameBuffers,
-  AudioBuffer,
-} from "./video_audio_preprocessor";
+} from "./VideoProcessor/video_audio_preprocessor";
 
 class PreviewVideoGeneration {
-  conditionedFrameBuffer: EngineFrameBuffers;
   mediaUploadAPI: MediaUploadApi;
   videoAPI: VideoApi;
   videoAndAudioPreProcessor: VideoAudioPreProcessor;
@@ -21,12 +17,6 @@ class PreviewVideoGeneration {
     onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void,
   ) {
     this.onProgress = onProgress;
-    this.conditionedFrameBuffer = {
-      colorFrames: [],
-      normalFrames: [],
-      outlineFrames: [],
-      depthFrames: [],
-    };
     this.videoAndAudioPreProcessor = new VideoAudioPreProcessor(onProgress);
     this.mediaUploadAPI = new MediaUploadApi();
     this.videoAPI = new VideoApi();
@@ -45,14 +35,7 @@ class PreviewVideoGeneration {
     // start polling for percentage change and convert and remap to remainint %
   }
 
-  async clearFrameBuffers() {
-    this.conditionedFrameBuffer = {
-      colorFrames: [],
-      normalFrames: [],
-      outlineFrames: [],
-      depthFrames: [],
-    };
-  }
+  async clearFrameBuffers() {}
 }
 
 export { PreviewVideoGeneration };

app/pages/PageEnigma/Editor/scene.ts

@@ -9,7 +9,7 @@ import { generateUUID } from "three/src/math/MathUtils.js";
 import { LoadingPlaceHolderManager } from "./placeholder_manager";
 import { MMDAnimationHelper, Water } from "three/examples/jsm/Addons.js";
 import { MediaFileType } from "../enums";
-import {ChromaKeyMaterial} from './chromakey'
+import { ChromaKeyMaterial } from "./chromakey";
 
 class Scene {
   name: string;
@@ -152,14 +152,20 @@ class Scene {
         //await videoElement.play();
         this.video_planes.push(videoElement);
 
-
         texture = new THREE.VideoTexture(videoElement);
 
         texture.colorSpace = THREE.SRGBColorSpace;
-        const image_material = new ChromaKeyMaterial(texture, 0x00ff00, 1920, 1080, 0.159, 0.082, 0.0);
+        const image_material = new ChromaKeyMaterial(
+          texture,
+          0x00ff00,
+          1920,
+          1080,
+          0.159,
+          0.082,
+          0.0,
+        );
         obj = new THREE.Mesh(geometry, image_material);
         obj.userData["media_id"] = name;
-
       } else {
         const loader = new THREE.TextureLoader();
         texture = loader.load(image_token);
@@ -371,6 +377,7 @@ class Scene {
     }
   }
 
+  // TODO: REPLACE
   async getMediaURL(media_id: string) {
     //This is for prod when we have the proper info on the url.
     const api_base_url = environmentVariables.values.BASE_API;
@@ -426,7 +433,7 @@ class Scene {
   }
 
   setVisible(object_uuid: string, visible: boolean) {
-    let object = this.get_object_by_uuid(object_uuid);
+    const object = this.get_object_by_uuid(object_uuid);
     if (object) {
       object.visible = visible;
       object.userData["visible"] = object.visible;
@@ -596,7 +603,7 @@ class Scene {
       child.userData["specular"] = 0.5;
       child.userData["locked"] = false;
       child.userData["media_file_type"] = MediaFileType.GLB;
-      
+
       child.layers.enable(0);
       child.layers.enable(1);
       if (load_version <= 1.0) {
@@ -678,13 +685,12 @@ class Scene {
     media_url: string,
     progress: (event: ProgressEvent) => void,
   ): Promise<THREE.SkinnedMesh> {
-
     // TODO: When converted to ts remove this and make ammo await instead.
     await this.delay_mmd(100);
 
-    console.log('Load MMD')
-    const scriptModule = document.createElement('script');
-    scriptModule.type = 'module';
+    console.log("Load MMD");
+    const scriptModule = document.createElement("script");
+    scriptModule.type = "module";
 
     scriptModule.textContent = `
       Ammo().then(function (AmmoLib) {
@@ -699,7 +705,8 @@ class Scene {
     return new Promise((resolve, reject) => {
       const mmdLoader = new MMDLoader();
       mmdLoader.loadWithAnimation(
-        media_url, ["/resources/pose/Lumine Idle cycle.vmd"],
+        media_url,
+        ["/resources/pose/Lumine Idle cycle.vmd"],
         (mmd) => {
           this.helper.add(mmd.mesh, {
             animation: mmd.animation,

app/pages/PageEnigma/Editor/timeline.ts

@@ -240,7 +240,7 @@ export class TimeLine {
           await this.editorEngine.generateVideo();
           this.debounce_generate_video = false;
         }
-        console.log("Finished! Generate Video");
+        console.log("Generate Video Press Event");
         break;
       }
       case toEngineActions.CHANGE_CAMERA_ASPECT_RATIO: {
@@ -755,9 +755,11 @@ export class TimeLine {
       // Caps to 10fps so that the buffering issue is solved and it plays i am not sure how to fix this.
       // TODO: Fix buffering and make 30 fps.
       const fixedPoint = 1;
-      
+
       let pb = parseFloat(
-        (this.scrubber_frame_position / this.editorEngine.cap_fps).toFixed(fixedPoint),
+        (this.scrubber_frame_position / this.editorEngine.cap_fps).toFixed(
+          fixedPoint,
+        ),
       );
       pb = parseFloat((pb % video_plane.duration).toFixed(fixedPoint));
       if (video_plane.currentTime !== pb) {
@@ -847,9 +849,6 @@ export class TimeLine {
       }
     }
 
-
-    //this.scene.helper.update(0.1);
-
     if (
       this.scrubber_frame_position >= this.timeline_limit &&
       this.is_playing

app/pages/PageEnigma/Editor/video_generation.ts

@@ -5,8 +5,7 @@ import Queue from "~/pages/PageEnigma/Queue/Queue";
 import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
 import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
 import { ToastTypes, ClipType } from "~/enums";
-import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
-import { ClipUI } from "../datastructures/clips/clip_ui.js";
+
 import { Visibility } from "./api_manager.js";
 import * as THREE from "three";
 import { getSceneSignals, addToast, startPollingActiveJobs } from "~/signals";
@@ -14,15 +13,15 @@ import { v4 as uuidv4 } from "uuid";
 import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
 import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager";
 import { globalIPAMediaToken } from "../signals";
-import { VideoPreProcessorError } from "./video_audio_preprocessor";
+import { BufferType } from "./VideoProcessor/engine_buffer";
 
-// TODO THIS CLASS MAKES NO SENSE Refactor so we generate all the frames first. then pass it through this pipeline as a data structure process it. through this class.
+// TODO THIS CLASS MAKES NO SENSE
+// Refactor so we generate all the frames first.
+// then pass it through this pipeline as a data structure process it.
+// through this class.
 
 interface MediaTokens {
   color: string;
-  normal: string;
-  depth: string;
-  outline: string;
 }
 
 export class VideoGeneration {
@@ -42,7 +41,7 @@ export class VideoGeneration {
     this.mediaUploadAPI = new MediaUploadApi();
     this.videoAPI = new VideoApi();
     this.last_scene_check_sum = "";
-    this.last_media_tokens = { color: "", normal: "", depth: "", outline: "" };
+    this.last_media_tokens = { color: "" };
     this.last_position_of_preprocessing = false;
   }
 
@@ -50,6 +49,54 @@ export class VideoGeneration {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }
 
+  async videoProcessingPipeline(title: string): Promise<MediaTokens> {
+    const mediaTokens: MediaTokens = {
+      color: "",
+    };
+    try {
+      // TODO send the audio a later time.
+      // const audioClips = this.editor.timeline.timeline_items.filter((clip) => {
+      //   return clip.type == ClipType.AUDIO;
+      // });
+
+      if (!this.editor.engineFrameBuffers.payloadZip) {
+        await this.handleError(
+          `Video Processing Failed Please Try Again (zip process failed)`,
+          3000,
+        );
+      }
+
+      const zipBlob = this.editor.engineFrameBuffers.payloadZip;
+      if (!zipBlob) {
+        await this.handleError(`"Payload blob is null."`, 3000);
+      }
+
+      const video_upload_response = await this.mediaUploadAPI.UploadStudioShot({
+        maybe_title: title,
+        uuid_idempotency_token: uuidv4(),
+        blob: zipBlob,
+        fileName: `${title}.zip`,
+        maybe_visibility: Visibility.Public,
+      });
+
+      if (!video_upload_response.data) {
+        console.log("Missing media_tokens.color Token");
+        const error = {
+          message: "Missing Data From Video Upload Response",
+          code: 1,
+        };
+        throw error;
+      }
+
+      mediaTokens.color = video_upload_response.data;
+      console.log(`https://storyteller.ai/media_files/${mediaTokens.color}`);
+    } finally {
+      await this.editor.engineFrameBuffers.clearBuffer(BufferType.COLOR);
+    }
+
+    return mediaTokens;
+  }
+
   // this is to generate previews
   async generateFrame() {
     if (!this.editor.generating_preview && this.editor.rawRenderer) {
@@ -62,11 +109,6 @@ export class VideoGeneration {
         });
       }
 
-      // if (this.editor.render_camera && this.editor.cam_obj) {
-      //   this.editor.render_camera.position.copy(this.editor.cam_obj.position);
-      //   this.editor.render_camera.rotation.copy(this.editor.cam_obj.rotation);
-      // }
-
       previewSrc.value = "";
 
       this.editor.rawRenderer.setSize(
@@ -145,13 +187,6 @@ export class VideoGeneration {
 
     // precondition checks, if we have no frames then we shouldn't try to do generations or snapshots
     // This means no frames so error out
-    if (this.editor.engineFrameBuffers.colorFrames.length <= 0) {
-      await this.handleError(
-        "Failed to Render Nothing to Animate in the Scene.",
-        3000,
-      );
-      return;
-    }
 
     this.editor.generating_preview = true;
 
@@ -161,11 +196,8 @@ export class VideoGeneration {
         "Please stay on this screen and do not switch tabs! while your video is being processed.",
     });
 
-    const media_tokens: MediaTokens = {
+    let media_tokens: MediaTokens = {
       color: "",
-      normal: "",
-      depth: "",
-      outline: "",
     };
 
     // properties used by the external scope to the for loop.
@@ -266,201 +298,15 @@ export class VideoGeneration {
       message:
         "Please stay on this screen and do not switch tabs! while your video is being processed.",
     });
-
+    // write the end point to upload the video
+    // upload the color frames with end point.
     try {
-      // Sound is only on the color not the other maps
-      const colorBlob =
-        await this.editor.videoAudioPreProcessor.processVideoWithBuffer(
-          this.editor.engineFrameBuffers.colorFrames,
-          "color",
-          "color",
-          "colorFrame",
-        );
-
-      await this.editor.videoAudioPreProcessor.removeInMemoryImages();
-      //await this.editor.videoAudioPreProcessor.debugListFiles();
-
-      let shouldProcessAudio = false;
-      const audioClips = this.editor.timeline.timeline_items.filter((clip) => {
-        return clip.type == ClipType.AUDIO;
-      });
-      // If there are any audio clips continue processing otherwise this whole process of combining the video
-      if (audioClips.length > 0) {
-        shouldProcessAudio = true;
-      }
-
-      if (shouldProcessAudio) {
-        const audioBuffer =
-          await this.editor.videoAudioPreProcessor.audioClipsToAudioBuffer(
-            audioClips,
-          );
-
-        await this.editor.videoAudioPreProcessor.processAudioWithBuffer(
-          audioBuffer,
-          "wav_track",
-          false,
-        );
-
-        // debug
-        //await this.editor.videoAudioPreProcessor.outputWav("wav_track");
-        const combinedVideoBlob =
-          await this.editor.videoAudioPreProcessor.combineAudioAndVideo(
-            "wav_track",
-            "color",
-            "output", // in memory output
-          );
-
-        // debug
-        //await this.editor.videoAudioPreProcessor.outputMp4("output");
-
-        const video_upload_response = await this.mediaUploadAPI.UploadNewVideo({
-          blob: combinedVideoBlob,
-          fileName: `${title}.mp4`,
-          uuid: uuidv4(),
-          maybe_title: title,
-          maybe_visibility: Visibility.Public,
-          maybe_style_name: style_name,
-          maybe_scene_source_media_file_token: immutable_media_token,
-        });
-
-        if (!video_upload_response.data) {
-          console.log("Missing media_tokens.color Token");
-          await this.handleError("Failed to Process to Server", 3000);
-          return;
-        }
-
-        media_tokens.color = video_upload_response.data;
-        console.log(`https://storyteller.ai/media/${media_tokens.color}`);
-        console.log("Uploaded with Audio");
-      } else {
-        // no audio
-        const video_upload_response = await this.mediaUploadAPI.UploadNewVideo({
-          blob: colorBlob,
-          fileName: `${title}.mp4`,
-          uuid: uuidv4(),
-          maybe_title: title,
-          maybe_visibility: Visibility.Public,
-          maybe_style_name: style_name,
-          maybe_scene_source_media_file_token: immutable_media_token,
-        });
-
-        if (!video_upload_response.data) {
-          await this.handleError("Failed to Process to Server", 3000);
-          return;
-        }
-
-        media_tokens.color = video_upload_response.data;
-        console.log(`https://storyteller.ai/media/${media_tokens.color}`);
-        console.log("Uploaded No Audio");
-      }
+      media_tokens = await this.videoProcessingPipeline(title);
     } catch (error) {
-      const err = error as VideoPreProcessorError;
-      console.log(error);
-      await this.handleError(
-        `Failed PreProcess Operation: ${err.message}`,
-        3000,
-      );
-    } finally {
-      // Remove all images in memory for color
-      await this.editor.videoAudioPreProcessor.removeInMemoryImages();
-      this.editor.engineFrameBuffers.colorFrames = []; // possibly allocate up front
+      await this.handleError(`Video Processing Failed Please Try Again`, 3000);
+      throw error;
     }
 
-    if (this.editor.engine_preprocessing) {
-      try {
-        const depthBlob = await this.enginePreProcessingStep(
-          "depth",
-          this.editor.engineFrameBuffers.depthFrames,
-        );
-
-        const depth_video_upload_response =
-          await this.mediaUploadAPI.UploadNewVideo({
-            blob: depthBlob,
-            fileName: `${title}.mp4`,
-            uuid: uuidv4(),
-            maybe_title: title,
-            maybe_visibility: Visibility.Public,
-            maybe_style_name: style_name,
-            maybe_scene_source_media_file_token: immutable_media_token,
-          });
-
-        if (!depth_video_upload_response.data) {
-          this.handleError("Failed to Upload PreProcessing 9D", 3000);
-          return;
-        }
-        media_tokens.depth = depth_video_upload_response.data;
-
-        const outlineBlob = await await this.enginePreProcessingStep(
-          "outline",
-          this.editor.engineFrameBuffers.outlineFrames,
-        );
-
-        const outline_video_upload_response =
-          await this.mediaUploadAPI.UploadNewVideo({
-            blob: outlineBlob,
-            fileName: `${title}.mp4`,
-            uuid: uuidv4(),
-            maybe_title: title,
-            maybe_visibility: Visibility.Public,
-            maybe_style_name: style_name,
-            maybe_scene_source_media_file_token: immutable_media_token,
-          });
-
-        if (!outline_video_upload_response.data) {
-          this.handleError("Failed to Upload PreProcessing 9O", 3000);
-          return;
-        }
-        media_tokens.outline = outline_video_upload_response.data;
-
-        const normalBlob = await this.enginePreProcessingStep(
-          "normal",
-          this.editor.engineFrameBuffers.normalFrames,
-        );
-
-        const normal_video_upload_response =
-          await this.mediaUploadAPI.UploadNewVideo({
-            blob: normalBlob,
-            fileName: `${title}.mp4`,
-            uuid: uuidv4(),
-            maybe_title: title,
-            maybe_visibility: Visibility.Public,
-            maybe_style_name: style_name,
-            maybe_scene_source_media_file_token: immutable_media_token,
-          });
-
-        if (!normal_video_upload_response.data) {
-          this.handleError("Failed to Upload PreProcessing 9N", 3000);
-          return;
-        }
-        media_tokens.normal = normal_video_upload_response.data;
-      } catch (err) {
-        console.log(err);
-        console.log("Could Not Engine PreProcess");
-        // At any point if it fails we need to clear and clean up tokens
-        media_tokens.depth = "";
-        media_tokens.outline = "";
-        media_tokens.outline = "";
-      } finally {
-        this.editor.engineFrameBuffers.normalFrames = [];
-        this.editor.engineFrameBuffers.outlineFrames = [];
-        this.editor.engineFrameBuffers.depthFrames = [];
-
-        console.log(
-          `https://storyteller.ai/media/${media_tokens.normal} Normal`,
-        );
-
-        console.log(`https://storyteller.ai/media/${media_tokens.depth} Depth`);
-
-        console.log(
-          `https://storyteller.ai/media/${media_tokens.outline} Outline`,
-        );
-
-        await this.editor.videoAudioPreProcessor.removeInMemoryImages();
-      }
-    }
-
-    await this.editor.videoAudioPreProcessor.removeAllVideos();
-
     this.editor.onWindowResize();
 
     this.editor.setColorMap();
@@ -475,26 +321,9 @@ export class VideoGeneration {
     this.last_media_tokens = media_tokens;
 
     await this.handleEnqueue(media_tokens, ipa_image_token ?? "");
-
     await this.EndLoadingState();
   }
 
-  // returns the media token
-  async enginePreProcessingStep(
-    frameType: string,
-    frameBuffer: (string | null)[],
-  ): Promise<Blob> {
-    const preProcessBlob =
-      await this.editor.videoAudioPreProcessor.processVideoWithBuffer(
-        frameBuffer,
-        `${frameType}`,
-        `${frameType}`,
-        `${frameType}Frame`,
-      );
-    await this.editor.videoAudioPreProcessor.removeInMemoryImages();
-    return preProcessBlob;
-  }
-
   async handleCachedEnqueue() {
     const enqueue_studio_response = await this.videoAPI.EnqueueStudio({
       enqueueVideo: {
@@ -515,9 +344,9 @@ export class VideoGeneration {
         use_upscaler: this.editor.generation_options.upscale,
         uuid_idempotency_token: uuidv4(),
         global_ipa_media_token: globalIPAMediaToken.value ?? "",
-        input_depth_file: this.last_media_tokens.depth,
-        input_normal_file: this.last_media_tokens.normal,
-        input_outline_file: this.last_media_tokens.outline,
+        input_depth_file: "",
+        input_normal_file: "",
+        input_outline_file: "",
         creator_set_visibility: Visibility.Public,
       },
     });
@@ -555,9 +384,9 @@ export class VideoGeneration {
         use_upscaler: this.editor.generation_options.upscale,
         uuid_idempotency_token: uuidv4(),
         global_ipa_media_token: ipa_image_token,
-        input_depth_file: upload_tokens.depth,
-        input_normal_file: upload_tokens.normal,
-        input_outline_file: upload_tokens.outline,
+        input_depth_file: "",
+        input_normal_file: "",
+        input_outline_file: "",
         creator_set_visibility: Visibility.Public,
       },
     });

app/root.tsx

@@ -57,6 +57,7 @@ export const links: LinksFunction = () => [
   },
 ];
 
+// .env part 2 add to this
 export async function loader() {
   const env = {
     // @ts-expect-error ProvessEnv is correct
@@ -83,6 +84,10 @@ export async function loader() {
     CONTEXT: process.env.CONTEXT || "%CONTEXT%",
     // @ts-expect-error ProvessEnv is correct
     DEPLOY_CONTEXT: process.env.DEPLOY_CONTEXT || "%DEPLOY_CONTEXT%",
+
+    // .env part 3
+    // @ts-expect-error ProvessEnv is correct
+    UPLOAD_API_VIDEO: process.env.UPLOAD_API_VIDEO || "%UPLOAD_API_VIDEO%",
   } as Record<string, string | boolean>;
   return { ENV: env };
 }

package-lock.json

@@ -7,9 +7,6 @@
       "name": "gottagofast_v0.0.0",
       "dependencies": {
         "@awesome.me/kit-fde2be5eb0": "^1.0.7",
-        "@ffmpeg/core": "0.11.0",
-        "@ffmpeg/ffmpeg": "0.11.6",
-        "@ffmpeg/util": "0.12.0",
         "@fortawesome/fontawesome-svg-core": "^6.5.1",
         "@fortawesome/free-brands-svg-icons": "^6.5.2",
         "@fortawesome/pro-solid-svg-icons": "^6.5.1",
@@ -26,6 +23,7 @@
         "@remix-run/serve": "^2.8.1",
         "@tweenjs/tween.js": "^18.6.4",
         "@types/lodash": "^4.17.6",
+        "@zip.js/zip.js": "^2.7.51",
         "ammo.js": "^0.0.10",
         "ccapture.js": "github:aceslowman/ccapture.js#npm-fix",
         "common": "file:common",
@@ -93,7 +91,7 @@
         "terser": "^5.30.3",
         "ts-jest": "^29.1.4",
         "typescript": "^5.1.6",
-        "vite": "^5.0.0",
+        "vite": "^5.4.0",
         "vite-tsconfig-paths": "^4.2.1"
       },
       "engines": {
@@ -2846,38 +2844,6 @@
       "integrity": "sha512-cEee/Z+I12mZcFJshKcCqC8tuX5hG3s+d+9nZ3LabqKF1vKdF41B92pJVCBggjAGORAeOzyyDDKrZwIkLffeOQ==",
       "dev": true
     },
-    "node_modules/@ffmpeg/core": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.11.0.tgz",
-      "integrity": "sha512-9Tt/+2PMpkGPXUK8n6He9G8Y+qR6qmCPSCw9iEKZxHHOvJ9BE/r0Fccj+YgDZTlyu6rXxc9x6EqCaFBIt7qzjA=="
-    },
-    "node_modules/@ffmpeg/ffmpeg": {
-      "version": "0.11.6",
-      "resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.11.6.tgz",
-      "integrity": "sha512-uN8J8KDjADEavPhNva6tYO9Fj0lWs9z82swF3YXnTxWMBoFLGq3LZ6FLlIldRKEzhOBKnkVfA8UnFJuvGvNxcA==",
-      "dependencies": {
-        "is-url": "^1.2.4",
-        "node-fetch": "^2.6.1",
-        "regenerator-runtime": "^0.13.7",
-        "resolve-url": "^0.2.1"
-      },
-      "engines": {
-        "node": ">=12.16.1"
-      }
-    },
-    "node_modules/@ffmpeg/ffmpeg/node_modules/regenerator-runtime": {
-      "version": "0.13.11",
-      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
-      "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg=="
-    },
-    "node_modules/@ffmpeg/util": {
-      "version": "0.12.0",
-      "resolved": "https://registry.npmjs.org/@ffmpeg/util/-/util-0.12.0.tgz",
-      "integrity": "sha512-8sHCW8H/ngqVhbRvCCX4e4uDNgZVoz8uNRry1zphIyzdX6flfBa2TmVwJ4g9/Qw//eAubEnuHaSxDGWWxcOTjg==",
-      "engines": {
-        "node": ">=18.17.0"
-      }
-    },
     "node_modules/@fortawesome/fontawesome-common-types": {
       "version": "6.5.1",
       "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.5.1.tgz",
@@ -8112,6 +8078,16 @@
         "node": ">=12 <14 || 14.2 - 14.9 || >14.10.0"
       }
     },
+    "node_modules/@zip.js/zip.js": {
+      "version": "2.7.51",
+      "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.7.51.tgz",
+      "integrity": "sha512-RKHaebzZZgQkUuzb49/qweN69e8Np9AUZ9QygydDIrbG1njypSAKwkeqIVeuf2JVGBDyB7Z9HKvzPgYrSlv9gw==",
+      "engines": {
+        "bun": ">=0.7.0",
+        "deno": ">=1.0.0",
+        "node": ">=16.5.0"
+      }
+    },
     "node_modules/@zxing/text-encoding": {
       "version": "0.9.0",
       "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz",
@@ -10592,6 +10568,7 @@
       "version": "0.1.13",
       "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
       "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
+      "dev": true,
       "optional": true,
       "dependencies": {
         "iconv-lite": "^0.6.2"
@@ -10601,6 +10578,7 @@
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
       "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "dev": true,
       "optional": true,
       "dependencies": {
         "safer-buffer": ">= 2.1.2 < 3.0.0"
@@ -13985,11 +13963,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/is-url": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
-      "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww=="
-    },
     "node_modules/is-weakmap": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz",
@@ -17841,6 +17814,7 @@
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
       "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "dev": true,
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
@@ -19004,9 +18978,9 @@
       }
     },
     "node_modules/picocolors": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
-      "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ=="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
+      "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
     },
     "node_modules/picomatch": {
       "version": "2.3.1",
@@ -19093,9 +19067,9 @@
       }
     },
     "node_modules/postcss": {
-      "version": "8.4.38",
-      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz",
-      "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==",
+      "version": "8.4.41",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz",
+      "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==",
       "devOptional": true,
       "funding": [
         {
@@ -19113,7 +19087,7 @@
       ],
       "dependencies": {
         "nanoid": "^3.3.7",
-        "picocolors": "^1.0.0",
+        "picocolors": "^1.0.1",
         "source-map-js": "^1.2.0"
       },
       "engines": {
@@ -20722,12 +20696,6 @@
         "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
       }
     },
-    "node_modules/resolve-url": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
-      "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==",
-      "deprecated": "https://github.com/lydell/resolve-url#deprecated"
-    },
     "node_modules/resolve.exports": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz",
@@ -22461,7 +22429,8 @@
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+      "dev": true
     },
     "node_modules/tree-kill": {
       "version": "1.2.2",
@@ -23824,13 +23793,13 @@
       }
     },
     "node_modules/vite": {
-      "version": "5.2.8",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.8.tgz",
-      "integrity": "sha512-OyZR+c1CE8yeHw5V5t59aXsUPPVTHMDjEZz8MgguLL/Q7NblxhZUlTu9xSPqlsUO/y+X7dlU05jdhvyycD55DA==",
+      "version": "5.4.0",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz",
+      "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==",
       "devOptional": true,
       "dependencies": {
-        "esbuild": "^0.20.1",
-        "postcss": "^8.4.38",
+        "esbuild": "^0.21.3",
+        "postcss": "^8.4.40",
         "rollup": "^4.13.0"
       },
       "bin": {
@@ -23850,6 +23819,7 @@
         "less": "*",
         "lightningcss": "^1.21.0",
         "sass": "*",
+        "sass-embedded": "*",
         "stylus": "*",
         "sugarss": "*",
         "terser": "^5.4.0"
@@ -23867,6 +23837,9 @@
         "sass": {
           "optional": true
         },
+        "sass-embedded": {
+          "optional": true
+        },
         "stylus": {
           "optional": true
         },
@@ -23919,10 +23892,26 @@
         }
       }
     },
+    "node_modules/vite/node_modules/@esbuild/aix-ppc64": {
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz",
+      "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==",
+      "cpu": [
+        "ppc64"
+      ],
+      "dev": true,
+      "optional": true,
+      "os": [
+        "aix"
+      ],
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/vite/node_modules/@esbuild/android-arm": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.2.tgz",
-      "integrity": "sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz",
+      "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==",
       "cpu": [
         "arm"
       ],
@@ -23936,9 +23925,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/android-arm64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz",
-      "integrity": "sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz",
+      "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==",
       "cpu": [
         "arm64"
       ],
@@ -23952,9 +23941,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/android-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.2.tgz",
-      "integrity": "sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz",
+      "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==",
       "cpu": [
         "x64"
       ],
@@ -23968,9 +23957,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/darwin-arm64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz",
-      "integrity": "sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz",
+      "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==",
       "cpu": [
         "arm64"
       ],
@@ -23984,9 +23973,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/darwin-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz",
-      "integrity": "sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz",
+      "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==",
       "cpu": [
         "x64"
       ],
@@ -24000,9 +23989,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/freebsd-arm64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz",
-      "integrity": "sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz",
+      "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==",
       "cpu": [
         "arm64"
       ],
@@ -24016,9 +24005,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/freebsd-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz",
-      "integrity": "sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz",
+      "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==",
       "cpu": [
         "x64"
       ],
@@ -24032,9 +24021,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-arm": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz",
-      "integrity": "sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz",
+      "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==",
       "cpu": [
         "arm"
       ],
@@ -24048,9 +24037,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-arm64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz",
-      "integrity": "sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz",
+      "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==",
       "cpu": [
         "arm64"
       ],
@@ -24064,9 +24053,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-ia32": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz",
-      "integrity": "sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz",
+      "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==",
       "cpu": [
         "ia32"
       ],
@@ -24080,9 +24069,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-loong64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz",
-      "integrity": "sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz",
+      "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==",
       "cpu": [
         "loong64"
       ],
@@ -24096,9 +24085,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-mips64el": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz",
-      "integrity": "sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz",
+      "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==",
       "cpu": [
         "mips64el"
       ],
@@ -24112,9 +24101,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-ppc64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz",
-      "integrity": "sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz",
+      "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==",
       "cpu": [
         "ppc64"
       ],
@@ -24128,9 +24117,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-riscv64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz",
-      "integrity": "sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz",
+      "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==",
       "cpu": [
         "riscv64"
       ],
@@ -24144,9 +24133,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-s390x": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz",
-      "integrity": "sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz",
+      "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==",
       "cpu": [
         "s390x"
       ],
@@ -24160,9 +24149,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/linux-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz",
-      "integrity": "sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz",
+      "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==",
       "cpu": [
         "x64"
       ],
@@ -24176,9 +24165,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/netbsd-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz",
-      "integrity": "sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz",
+      "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==",
       "cpu": [
         "x64"
       ],
@@ -24192,9 +24181,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/openbsd-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz",
-      "integrity": "sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz",
+      "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==",
       "cpu": [
         "x64"
       ],
@@ -24208,9 +24197,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/sunos-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz",
-      "integrity": "sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz",
+      "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==",
       "cpu": [
         "x64"
       ],
@@ -24224,9 +24213,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/win32-arm64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz",
-      "integrity": "sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz",
+      "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==",
       "cpu": [
         "arm64"
       ],
@@ -24240,9 +24229,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/win32-ia32": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz",
-      "integrity": "sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz",
+      "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==",
       "cpu": [
         "ia32"
       ],
@@ -24256,9 +24245,9 @@
       }
     },
     "node_modules/vite/node_modules/@esbuild/win32-x64": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz",
-      "integrity": "sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz",
+      "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==",
       "cpu": [
         "x64"
       ],
@@ -24272,9 +24261,9 @@
       }
     },
     "node_modules/vite/node_modules/esbuild": {
-      "version": "0.20.2",
-      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz",
-      "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==",
+      "version": "0.21.5",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz",
+      "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==",
       "devOptional": true,
       "hasInstallScript": true,
       "bin": {
@@ -24284,29 +24273,29 @@
         "node": ">=12"
       },
       "optionalDependencies": {
-        "@esbuild/aix-ppc64": "0.20.2",
-        "@esbuild/android-arm": "0.20.2",
-        "@esbuild/android-arm64": "0.20.2",
-        "@esbuild/android-x64": "0.20.2",
-        "@esbuild/darwin-arm64": "0.20.2",
-        "@esbuild/darwin-x64": "0.20.2",
-        "@esbuild/freebsd-arm64": "0.20.2",
-        "@esbuild/freebsd-x64": "0.20.2",
-        "@esbuild/linux-arm": "0.20.2",
-        "@esbuild/linux-arm64": "0.20.2",
-        "@esbuild/linux-ia32": "0.20.2",
-        "@esbuild/linux-loong64": "0.20.2",
-        "@esbuild/linux-mips64el": "0.20.2",
-        "@esbuild/linux-ppc64": "0.20.2",
-        "@esbuild/linux-riscv64": "0.20.2",
-        "@esbuild/linux-s390x": "0.20.2",
-        "@esbuild/linux-x64": "0.20.2",
-        "@esbuild/netbsd-x64": "0.20.2",
-        "@esbuild/openbsd-x64": "0.20.2",
-        "@esbuild/sunos-x64": "0.20.2",
-        "@esbuild/win32-arm64": "0.20.2",
-        "@esbuild/win32-ia32": "0.20.2",
-        "@esbuild/win32-x64": "0.20.2"
+        "@esbuild/aix-ppc64": "0.21.5",
+        "@esbuild/android-arm": "0.21.5",
+        "@esbuild/android-arm64": "0.21.5",
+        "@esbuild/android-x64": "0.21.5",
+        "@esbuild/darwin-arm64": "0.21.5",
+        "@esbuild/darwin-x64": "0.21.5",
+        "@esbuild/freebsd-arm64": "0.21.5",
+        "@esbuild/freebsd-x64": "0.21.5",
+        "@esbuild/linux-arm": "0.21.5",
+        "@esbuild/linux-arm64": "0.21.5",
+        "@esbuild/linux-ia32": "0.21.5",
+        "@esbuild/linux-loong64": "0.21.5",
+        "@esbuild/linux-mips64el": "0.21.5",
+        "@esbuild/linux-ppc64": "0.21.5",
+        "@esbuild/linux-riscv64": "0.21.5",
+        "@esbuild/linux-s390x": "0.21.5",
+        "@esbuild/linux-x64": "0.21.5",
+        "@esbuild/netbsd-x64": "0.21.5",
+        "@esbuild/openbsd-x64": "0.21.5",
+        "@esbuild/sunos-x64": "0.21.5",
+        "@esbuild/win32-arm64": "0.21.5",
+        "@esbuild/win32-ia32": "0.21.5",
+        "@esbuild/win32-x64": "0.21.5"
       }
     },
     "node_modules/walker": {
@@ -24373,7 +24362,8 @@
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+      "dev": true
     },
     "node_modules/webm-writer": {
       "version": "1.0.0",
@@ -24399,6 +24389,7 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
       "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "dev": true,
       "dependencies": {
         "tr46": "~0.0.3",
         "webidl-conversions": "^3.0.0"

package.json

@@ -7,7 +7,7 @@
     "build": "remix vite:build",
     "dev": "remix vite:dev",
     "staging-build": "remix vite:build",
-    "lint": "eslint --ignore-path .gitignore --cache --cache-location ./node_modules/.cache/eslint .",
+    "lint": "eslint --ignore-path .gitignore --cache --cache-location ./node_modules/.cache/eslint playground/* .",
     "start": "netlify serve",
     "typecheck": "tsc",
     "update:icons": "npm update @awesome.me/kit-fde2be5eb0",
@@ -19,9 +19,6 @@
   },
   "dependencies": {
     "@awesome.me/kit-fde2be5eb0": "^1.0.7",
-    "@ffmpeg/core": "0.11.0",
-    "@ffmpeg/ffmpeg": "0.11.6",
-    "@ffmpeg/util": "0.12.0",
     "@fortawesome/fontawesome-svg-core": "^6.5.1",
     "@fortawesome/free-brands-svg-icons": "^6.5.2",
     "@fortawesome/pro-solid-svg-icons": "^6.5.1",
@@ -38,6 +35,7 @@
     "@remix-run/serve": "^2.8.1",
     "@tweenjs/tween.js": "^18.6.4",
     "@types/lodash": "^4.17.6",
+    "@zip.js/zip.js": "^2.7.51",
     "ammo.js": "^0.0.10",
     "ccapture.js": "github:aceslowman/ccapture.js#npm-fix",
     "common": "file:common",
@@ -105,7 +103,7 @@
     "terser": "^5.30.3",
     "ts-jest": "^29.1.4",
     "typescript": "^5.1.6",
-    "vite": "^5.0.0",
+    "vite": "^5.4.0",
     "vite-tsconfig-paths": "^4.2.1"
   },
   "engines": {
@@ -113,7 +111,7 @@
   },
   "version": "",
   "lint-staged": {
-    "*.js": "eslint --cache --fix",
+    "*.js": "eslint --cache --fix --ignore-pattern 'playground/**'",
     "*.{js,css,md}": "prettier --write"
   }
 }

playground/3jsDemo/ImageProcessingWorker.js

@@ -0,0 +1,76 @@
+import * as THREE from "three";
+
+export class FrameCollectorManager {
+  constructor(width, height, expectedFramesLength = 720) {
+    this.worker = new Worker("canvas-queued-worker.js");
+    this.worker.addEventListener("message", this.handleMessage.bind(this));
+
+    this.frames = [];
+
+    this.expectedFramesLength = expectedFramesLength;
+
+    this.offscreenCanvas = new OffscreenCanvas(width, height);
+    this.width = width;
+    this.height = height;
+    this.offScreenRenderer = new THREE.WebGLRenderer({
+      canvas: this.offscreenCanvas,
+    });
+  }
+
+  setCanvas(width, height) {
+    this.offscreenCanvas.width = width;
+    this.offscreenCanvas.height = height;
+  }
+
+  handleMessage(event) {
+    console.time("Decode");
+    console.log("Message from Decode:", event.data);
+    const { id, blob } = event.data;
+    console.log(`${id} Frame id: ${blob}`);
+    console.timeEnd("Decode");
+    this.frames.push(blob);
+    console.log(this.frames.length);
+
+    // create shared worker
+    if (this.frames.length === this.expectedFramesLength) {
+      console.log("Complete");
+    }
+  }
+
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async collectedFrames() {
+    while (this.frames.length != this.expectedFramesLength) {
+      await this.sleep(1);
+    }
+
+    return this.frames;
+  }
+
+  terminate() {
+    this.worker.terminate();
+    console.log("Worker Terminated");
+  }
+
+  render(scene, camera, frame) {
+    this.offScreenRenderer.render(scene, camera);
+    const imageBitmap = this.offscreenCanvas.transferToImageBitmap();
+    this.processFrame(imageBitmap, frame);
+  }
+
+  processFrame(imageBitmap, frame) {
+    const width = this.width;
+    const height = this.height;
+    const action = "process";
+
+    this.worker.postMessage({
+      action,
+      imageBitmap,
+      width,
+      height,
+      frame,
+    });
+  }
+}

playground/3jsDemo/canvas-queued-worker.js

@@ -0,0 +1,64 @@
+// This is responsible for processing and turning the data into
+
+class ImageProcessorWorker {
+  constructor() {
+    this.queue = [];
+    this.processing = false;
+    this.holdingCanvas = new OffscreenCanvas(0, 0);
+    this.ctx = this.holdingCanvas.getContext("bitmaprenderer");
+    this.imageType = "image/png";
+  }
+
+  enqueueTask(imageBitmap, width, height, id) {
+    this.queue.push({ imageBitmap, width, height, id });
+  }
+
+  async processQueue() {
+    // If already processing, return to avoid duplicate execution
+    console.log(`Current Queue Length ${this.queue.length}`);
+    if (this.queue.length > 0) {
+      const { imageBitmap, width, height, id } = this.queue.shift();
+      console.log(`Working On an Frame Id ${id}`);
+
+      this.holdingCanvas.width = width;
+      this.holdingCanvas.height = height;
+
+      try {
+        // Create or reuse the offscreen canvas
+        this.ctx.transferFromImageBitmap(imageBitmap);
+
+        // Convert the canvas content to a Blob
+        const blob = await this.holdingCanvas.convertToBlob({
+          quality: 1.0,
+          type: this.imageType,
+        });
+
+        // Send the Blob back to the main thread
+        self.postMessage({ blob, id });
+      } catch (error) {
+        console.error("Error processing image:", error);
+        self.postMessage({ error: error.message });
+      }
+    }
+
+    console.log("Processing");
+    setTimeout(this.processQueue.bind(this), 0);
+  }
+}
+
+// Instantiate the ImageProcessorWorker
+const runner = new ImageProcessorWorker();
+let start = false;
+
+self.addEventListener("message", async (event) => {
+  console.log(event.data);
+  const { type, imageBitmap, width, height, frame } = event.data;
+  console.log(`${type}: ${imageBitmap} ${width} ${height} ${frame}`);
+
+  if (start == false) {
+    start = true;
+    await runner.processQueue();
+  }
+
+  runner.enqueueTask(imageBitmap, width, height, frame);
+});

playground/3jsDemo/canvas-worker.js

@@ -0,0 +1,17 @@
+// Listen for messages from the main thread
+self.addEventListener("message", async function (event) {
+  const imageBitmap = event.data.imageBitmap;
+  const width = event.data.width;
+  const height = event.data.height;
+
+  const holdingCanvas = new OffscreenCanvas(width, height);
+  // in the worker use queue and pass the imageBitmap
+  const ctx = holdingCanvas.getContext("bitmaprenderer");
+  ctx.transferFromImageBitmap(imageBitmap);
+  const blob = await holdingCanvas.convertToBlob({
+    quality: 1.0,
+    type: "image/png",
+  });
+
+  self.postMessage(blob); // Send the result back to the main thread
+});

playground/3jsDemo/canvasworker.js

@@ -0,0 +1,89 @@
+class CanvasProcessor {
+  constructor() {
+    this.queue = [];
+    this.totalItems = 0;
+    this.completedItems = 0;
+    this.running = true;
+  }
+
+  // Add a canvas to the processing queue
+  enqueueCanvas(canvas, id) {
+    this.queue.push({ canvas, id });
+  }
+
+  // Set the total number of items to process
+  setTotalItems(count) {
+    this.totalItems = count;
+  }
+
+  // Start the processing loop
+  async processQueue() {
+    while (this.running) {
+      if (this.queue.length > 0) {
+        await this.processNext();
+      }
+
+      // Check if all items have been processed
+      if (this.completedItems >= this.totalItems) {
+        self.postMessage({ command: "done", message: "All items processed." });
+        this.running = false;
+      }
+
+      await this.sleep(100); // Small delay to prevent blocking
+    }
+  }
+
+  // Process the next canvas in the queue
+  async processNext() {
+    if (this.queue.length > 0) {
+      const { canvas, id } = this.queue.shift();
+
+      try {
+        const blob = await canvas.convertToBlob();
+
+        this.completedItems++;
+        const percentageComplete =
+          (this.completedItems / this.totalItems) * 100;
+
+        self.postMessage({
+          id,
+          blob,
+          completedItems: this.completedItems,
+          totalItems: this.totalItems,
+          percentageComplete,
+        });
+      } catch (error) {
+        self.postMessage({ id, error: error.message });
+      }
+    }
+  }
+
+  // Utility to sleep for a specified time
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+
+// Instantiate the processor
+const processor = new CanvasProcessor();
+
+// Listen for messages from the main thread
+self.addEventListener("message", function (event) {
+  const data = event.data;
+
+  if (data.totalItems) {
+    processor.setTotalItems(data.totalItems);
+  }
+
+  if (data.canvas) {
+    processor.enqueueCanvas(data.canvas, data.id);
+  }
+
+  if (data.command === "start") {
+    processor.processQueue();
+  }
+
+  if (data.command === "stop") {
+    processor.running = false;
+  }
+});

playground/3jsDemo/enginepreprocessing.js

@@ -0,0 +1,29 @@
+// if (this.engine_preprocessing) {
+//   // Normals
+//   this.setNormalMap();
+//   this.render_composer?.render();
+
+//   const normalRenderTask = this.videoAudioPreProcessor.retrieveFrame(
+//     this.rawRenderer,
+//   );
+//   this.engineFrameBuffers.normalFrames[this.renderIndex] =
+//     await normalRenderTask;
+//   // Depth
+//   this.setRenderDepth();
+//   this.render_composer?.render();
+//   const depthRenderTask = this.videoAudioPreProcessor.retrieveFrame(
+//     this.rawRenderer,
+//   );
+//   this.engineFrameBuffers.depthFrames[this.renderIndex] =
+//     await depthRenderTask;
+
+//   // Outline
+//   this.setOutlineRender();
+//   this.render_composer?.render();
+//   const outlineRenderTask = this.videoAudioPreProcessor.retrieveFrame(
+//     this.rawRenderer,
+//   );
+//   this.engineFrameBuffers.outlineFrames[this.renderIndex] =
+//     await outlineRenderTask;
+//   this.renderIndex += 1;
+// } else {

playground/3jsDemo/ffmpeg-worker.js

@@ -0,0 +1,121 @@
+// import { VideoGeneration } from "../../app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts";
+
+//import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
+self.importScripts("@ffmpeg/ffmpeg");
+const JPEG = "image/jpeg";
+const PNG = "image/png";
+
+// chrome://inspect/#workers debug shared worker.
+
+async function processVideoWithBuffer(
+  buffer,
+  fileName,
+  outFileName,
+  imagePrefix,
+) {
+  try {
+    const imageFormat = this.properties.imageFormat;
+    let extension = "";
+
+    if (imageFormat == JPEG) {
+      extension = ".jpeg";
+    } else {
+      extension = ".png";
+    }
+    // console.log("Image Files");
+    // console.log(`Buffer Lenght ${buffer.length}`);
+    for (let index = 0; index < buffer.length; index++) {
+      const element = buffer[index];
+      if (element === null) {
+        continue;
+      }
+      // this can fail so catch any errors.
+      const image = await fetchFile(element);
+      await this.ffmpeg.FS(
+        "writeFile",
+        `${imagePrefix}_image${index}${extension}`,
+        image,
+      );
+    }
+
+    await this.ffmpeg.run(
+      "-framerate",
+      "" + this.properties.frameRate,
+      "-i",
+      `${imagePrefix}_image%d${extension}`,
+      `${fileName}.mp4`,
+    );
+
+    await this.ffmpeg.run(
+      "-i",
+      `${fileName}.mp4`,
+      "-f",
+      "lavfi",
+      "-i",
+      "anullsrc", // This adds a silent audio track
+      "-max_muxing_queue_size",
+      "999999",
+      "-vf", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE
+      "select=gte(n\\,1)", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE Applies a video filter to remove the first frame.
+      // "select=gte(n\\,1),scale=1024:576",
+      "-c:v",
+      "libx264", // Specify video codec (optional, but recommended for MP4)
+      "-c:a",
+      "aac", // Specify audio codec (optional, but recommended for MP4)
+      "-shortest", // Ensure output duration matches the shortest stream (video or audio)
+      "-pix_fmt",
+      "yuv420p",
+      "-f",
+      "mp4",
+      `${outFileName}.mp4`,
+    );
+
+    const blob = await this.outputMp4FromMemory(`${outFileName}`);
+    return blob;
+  } catch (error) {
+    console.error(error);
+  }
+}
+
+self.addEventListener("connect", function (event) {
+  const port = event.ports[0];
+
+  port.addEventListener("message", function (event) {
+    console.log("Message received from main script:", event.data);
+    port.postMessage("Hello from the Shared Worker!");
+  });
+
+  port.start();
+});
+
+// const videoGeneration = new VideoGeneration();
+
+// self.addEventListener("connect", function (event) {
+//   const port = event.ports[0];
+
+//   console.log("WERE IN");
+//   port.start();
+
+//   port.addEventListener("message", async function (event) {
+//     console.log("Message received from main script:", event.data);
+
+//     const { colorFrames } = event.data;
+//     console.log(event.data);
+
+//     try {
+//       console.time("Processing Video Started");
+//       const colorBlob = await videoGeneration.processVideoWithBuffer(
+//         colorFrames,
+//         "color",
+//         "color",
+//         "colorFrame",
+//       );
+//       console.log(colorBlob);
+//       console.time("Processing Video Finished");
+//       port.postMessage({ colorBlob });
+//     } catch (error) {
+//       console.log(error);
+//       port.postMessage({ error: error.message });
+//     }
+//   });
+// });

playground/3jsDemo/index.html

@@ -0,0 +1,25 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>My first three.js app</title>
+    <style>
+      body {
+        margin: 0;
+      }
+    </style>
+    <script type="importmap">
+      {
+        "imports": {
+          "three": "https://cdn.jsdelivr.net/npm/three@v0.149.0/build/three.module.js",
+          "three/addons/": "https://cdn.jsdelivr.net/npm/three@v0.149.0/examples/jsm/",
+          "ffmpeg": "https://cdn.jsdelivr.net/npm/@ffmpeg/ffmpeg@0.12.10/dist/umd/ffmpeg.min.js"
+        }
+      }
+    </script>
+  </head>
+
+  <body>
+    <script type="module" src="/main.js"></script>
+  </body>
+</html>

playground/3jsDemo/main.js

@@ -0,0 +1,80 @@
+import * as THREE from "three";
+
+import { FrameCollectorManager } from "./ImageProcessingWorker.js";
+
+const scene = new THREE.Scene();
+const camera = new THREE.PerspectiveCamera(
+  75,
+  window.innerWidth / window.innerHeight,
+  0.1,
+  1000,
+);
+
+const renderer = new THREE.WebGLRenderer();
+renderer.setSize(window.innerWidth, window.innerHeight);
+renderer.setAnimationLoop(animate);
+document.body.appendChild(renderer.domElement);
+
+const geometry = new THREE.BoxGeometry(1, 1, 1);
+const material = new THREE.MeshBasicMaterial({ color: 0x00ff00 });
+const cube = new THREE.Mesh(geometry, material);
+scene.add(cube);
+
+camera.position.z = 5;
+
+const totalFrames = 720;
+
+const imageProcessing = new FrameCollectorManager(
+  window.innerWidth,
+  window.innerHeight,
+  totalFrames,
+);
+
+let frame = 0;
+
+// const sharedWorker = new SharedWorker("ffmpeg-worker.js");
+// // Start the port
+// sharedWorker.port.start();
+// const message = { message: "Hello" };
+// sharedWorker.port.postMessage(message);
+
+if (typeof SharedWorker !== "undefined") {
+  console.log("Shared Workers are supported.");
+  // You can safely create a Shared Worker here
+  const worker = new SharedWorker("ffmpeg-worker.js", { type: "module" });
+
+  // Get the port for communication
+  const port = worker.port;
+
+  // Listen for messages from the worker
+  port.addEventListener("message", function (event) {
+    console.log("Message received from worker:", event.data);
+  });
+
+  // Start the port
+  port.start();
+
+  // Send a message to the worker
+  port.postMessage("Hello from the main script!");
+} else {
+  console.log("Shared Workers are not supported in this browser.");
+}
+
+async function animate() {
+  cube.rotation.x += 0.01;
+  cube.rotation.y += 0.01;
+
+  frame += 1;
+  // buckets the frames.
+
+  if (frame <= totalFrames) {
+    //console.time("Render");
+    // can use the canvas renderer to enqueue as an option.
+    //imageProcessing.render(scene, camera, frame);
+    //console.timeEnd("Render");
+  }
+
+  // block and wait for all the frames
+
+  renderer.render(scene, camera);
+}

playground/3jsDemo/run.ps1

@@ -0,0 +1,10 @@
+# Run the TypeScript build
+npm run build
+
+# Check if the build was successful
+if ($?) {
+    # Open the index.html file in the default web browser
+    Start-Process "index.html"
+} else {
+    Write-Host "Build failed. Please check the errors above."
+}

playground/ts_playground.js

@@ -0,0 +1,35 @@
+class Network {
+  constructor() {}
+
+  async sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+  async task1() {
+    console.log("World.");
+    let val = true;
+    while (val) {
+      await this.sleep(1000);
+      console.log("Task1 doing work.");
+    }
+    //throw Error("Ded World.")
+  }
+
+  async task2() {
+    console.log("Hello.");
+    let val = true;
+    while (val) {
+      await this.sleep(1000);
+      console.log("Task2 doing work.");
+    }
+    //throw Error("Ded Hello.")
+  }
+}
+
+const network = new Network();
+
+async function main() {
+  network.task1();
+  network.task2();
+}
+
+main();

test/test_bench.js

@@ -1,38 +0,0 @@
-
-class Network {
-    constructor() { }
-
-    async sleep(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
-    }
-    async task1() {
-        console.log("World.")
-        let val = true
-        while (val) {
-            await this.sleep(1000); 
-            console.log("Task1 doing work.")
-        }
-        //throw Error("Ded World.")
-    }
-
-    async task2() {
-        console.log("Hello.")
-        let val = true 
-        while (val) {
-            await this.sleep(1000); 
-            console.log("Task2 doing work.")
-        }
-        //throw Error("Ded Hello.")
-    }
-}
-
-const network = new Network() 
-
-async function main() {
-    network.task1()
-    network.task2()
-}
-
-main()
-
-