From 04ec557da1322e564a5076bf1c0dd261f048f190 Mon Sep 17 00:00:00 2001 From: ArEnSc <6252325+ArEnSc@users.noreply.github.com> Date: Fri, 30 Aug 2024 19:22:37 -0400 Subject: [PATCH] Feature-Streaming Frames and FFMPEG (#671) Removed FF MPEG Added New Rendering code using web workers and zipping and uploading. Added a New endpoint to upload a bucket of frames. --- .env | 1 + app/Classes/ApiManager/ApiManager.ts | 3 + app/Classes/ApiManager/MediaUploadApi.ts | 99 ++++++ app/api/BucketConfig.ts | 1 + .../Editor/VideoProcessor/engine_buffer.ts | 261 ++++++++++++++ .../shared_workers/video_shared_worker.ts | 15 + .../video_shared_worker_manager.ts | 31 ++ .../video_audio_preprocessor.ts | 321 +++++++++--------- .../workers/frame_processor_worker.ts | 162 +++++++++ .../workers/frame_worker_manager.ts | 125 +++++++ app/pages/PageEnigma/Editor/editor.ts | 220 ++++++------ .../Editor/preview_video_generation.ts | 21 +- app/pages/PageEnigma/Editor/scene.ts | 29 +- app/pages/PageEnigma/Editor/timeline.ts | 11 +- .../PageEnigma/Editor/video_generation.ts | 305 ++++------------- app/root.tsx | 5 + package-lock.json | 297 ++++++++-------- package.json | 10 +- playground/3jsDemo/ImageProcessingWorker.js | 76 +++++ playground/3jsDemo/canvas-queued-worker.js | 64 ++++ playground/3jsDemo/canvas-worker.js | 17 + playground/3jsDemo/canvasworker.js | 89 +++++ playground/3jsDemo/enginepreprocessing.js | 29 ++ playground/3jsDemo/ffmpeg-worker.js | 121 +++++++ playground/3jsDemo/index.html | 25 ++ playground/3jsDemo/main.js | 80 +++++ playground/3jsDemo/run.ps1 | 10 + playground/ts_playground.js | 35 ++ test/test_bench.js | 38 --- 29 files changed, 1748 insertions(+), 753 deletions(-) create mode 100644 app/pages/PageEnigma/Editor/VideoProcessor/engine_buffer.ts create mode 100644 app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker.ts create mode 100644 app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts rename app/pages/PageEnigma/Editor/{ => VideoProcessor}/video_audio_preprocessor.ts (67%) create mode 100644 app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts create mode 100644 app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_worker_manager.ts create mode 100644 playground/3jsDemo/ImageProcessingWorker.js create mode 100644 playground/3jsDemo/canvas-queued-worker.js create mode 100644 playground/3jsDemo/canvas-worker.js create mode 100644 playground/3jsDemo/canvasworker.js create mode 100644 playground/3jsDemo/enginepreprocessing.js create mode 100644 playground/3jsDemo/ffmpeg-worker.js create mode 100644 playground/3jsDemo/index.html create mode 100644 playground/3jsDemo/main.js create mode 100644 playground/3jsDemo/run.ps1 create mode 100644 playground/ts_playground.js delete mode 100644 test/test_bench.js diff --git a/.env b/.env index 47ab30d0fb..f3a612888e 100644 --- a/.env +++ b/.env @@ -8,3 +8,4 @@ REACT_APP_PUBLIC_POSTHOG_KEY="phc_jBFgac0mVALAFk3negnSfYCcHgvkT00yBLQDmCDYNBb" REACT_APP_PUBLIC_POSTHOG_UI="https://us.i.posthog.com" DEPLOY_CONTEXT="DEVELOPMENT" CONTEXT="DEVELOPMENT" +UPLOAD_API_VIDEO="https://upload.storyteller.ai" diff --git a/app/Classes/ApiManager/ApiManager.ts b/app/Classes/ApiManager/ApiManager.ts index 489378fe14..48a847a884 100644 --- a/app/Classes/ApiManager/ApiManager.ts +++ b/app/Classes/ApiManager/ApiManager.ts @@ -13,12 +13,14 @@ export class ApiManager { ApiTargets: Record = {}; constructor() { + // look at the .env file this.ApiTargets = { BaseApi: environmentVariables.values.BASE_API as string, GoggleApi: environmentVariables.values.GOOGLE_API as string, FunnelApi: environmentVariables.values.FUNNEL_API as string, CdnApi: environmentVariables.values.CDN_API as string, GravatarApi: environmentVariables.values.GRAVATAR_API as string, + uploadApi: environmentVariables.values.UPLOAD_API_VIDEO as string, }; } @@ -152,6 +154,7 @@ export class ApiManager { formData.append("file", blob); } + console.log(`HELLO${formData}`); return this.fetchMultipartFormData(endpoint, { method: "POST", headers: { diff --git a/app/Classes/ApiManager/MediaUploadApi.ts b/app/Classes/ApiManager/MediaUploadApi.ts index d2b5120415..cab236b6a9 100644 --- a/app/Classes/ApiManager/MediaUploadApi.ts +++ b/app/Classes/ApiManager/MediaUploadApi.ts @@ -2,6 +2,105 @@ import { ApiManager, ApiResponse } from "./ApiManager"; import { FilterEngineCategories, Visibility } from "~/enums"; export class MediaUploadApi extends ApiManager { + private sessionToken: string = undefined; + + private async getSessionTokenForUploadStudioShot() { + const endpoint = `${this.ApiTargets.BaseApi}/v1/session_token`; + return await this.get<{ + maybe_signed_session: string; + }>({ endpoint }) + .then((response) => { + return { + success: true, + data: response["maybe_signed_session"], + }; + }) + .catch((err) => { + return { + success: false, + errorMessage: err.message, + }; + }); + } + + public async UploadStudioShot({ + maybe_title, + uuid_idempotency_token, + blob, + fileName, + maybe_visibility, + }: { + maybe_title?: string; + uuid_idempotency_token: string; + blob: Blob | null; + fileName: string; + maybe_visibility?: Visibility; + }): Promise> { + // This function is a bit special it requires that + const endpoint = `${this.ApiTargets.uploadApi}/v1/media_files/upload/studio_shot`; + + if (!this.sessionToken) { + console.log("Session Token"); + const response = await this.getSessionTokenForUploadStudioShot(); + if (response.success && response.data) { + this.sessionToken = response.data; + console.log(`Token ${this.sessionToken}`); + } else { + return { + success: false, + errorMessage: "Could not obtain session tokens.", + }; + } + } else { + console.log(`Session Token: ${this.sessionToken}`); + } + + const options: Record = { + maybe_title, + maybe_frame_rate_fps: 60, + maybe_visibility: maybe_visibility?.toString(), + }; + + const formRecord = Object.entries(options).reduce( + (allOptions, [key, value]) => { + if (value === undefined) { + return allOptions; + } + return { ...allOptions, [key]: value.toString() }; + }, + {} as Record, + ); + + const formData = new FormData(); + formData.append("uuid_idempotency_token", uuid_idempotency_token); + Object.entries(formRecord).forEach(([key, value]) => { + formData.append(key, value); + }); + if (blob && fileName) { + formData.append("file", blob, fileName); + } else if (blob) { + formData.append("file", blob); + } + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Accept: "application/json", + Session: this.sessionToken, + }, + credentials: "include", + body: formData, + }); + + const result = await response.json(); + + if (result.success && result.media_file_token) { + return { success: true, data: result.media_file_token }; + } else { + return { success: false, errorMessage: "Failed to Get Media Token" }; + } + } + private async Upload({ endpoint, uuid, diff --git a/app/api/BucketConfig.ts b/app/api/BucketConfig.ts index d5ccd64a68..5442e1d87a 100644 --- a/app/api/BucketConfig.ts +++ b/app/api/BucketConfig.ts @@ -11,6 +11,7 @@ class BucketConfig { } // TODO: Prevent callers with undefined/null paths + // TODO pipeline works make more robust.... getGcsUrl(bucketRelativePath: string | undefined | null): string { const bucket = this.getBucket(); let path = bucketRelativePath; diff --git a/app/pages/PageEnigma/Editor/VideoProcessor/engine_buffer.ts b/app/pages/PageEnigma/Editor/VideoProcessor/engine_buffer.ts new file mode 100644 index 0000000000..7a1d6d9af1 --- /dev/null +++ b/app/pages/PageEnigma/Editor/VideoProcessor/engine_buffer.ts @@ -0,0 +1,261 @@ +import { ImageFormat } from "./video_audio_preprocessor"; +import * as THREE from "three"; +import { FrameWorkerManager } from "./workers/frame_worker_manager"; + +import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js"; + +export interface IEngineFrameBuffers { + colorFrames: (Blob | null)[]; + clearBuffer(name: BufferType): Promise; + checkIfBufferIsEmpty(buffer: BufferType): Promise; + countNonNullFrames(buffer: BufferType): Promise; +} + +export enum BufferType { + COLOR = "colorFrames", +} + +export interface EngineFrameBuffersError { + message: string; + code: number; +} + +export class EngineFrameBuffers implements IEngineFrameBuffers { + colorFrames: (Blob | null)[]; + + private numberOfFrames: number; + public hasCache: boolean; + public colorRenderQueue: Promise[]; + private webWorkers: boolean; + + public payloadZip: Blob | null; + + public frameWorkerManager: FrameWorkerManager | undefined; + + // for the client side zip. + private zipFileWriter: BlobWriter | undefined; + private zipWriter: ZipWriter | undefined; + + private imageType = "image/jpeg"; + constructor(width: number, height: number) { + this.colorFrames = []; + this.colorRenderQueue = []; + this.payloadZip = null; + this.hasCache = false; + + this.zipFileWriter = undefined; + this.zipWriter = undefined; + + //Check for SharedWorker support + if (typeof Worker === "undefined") { + console.error("Web Workers is not supported in this browser."); + this.webWorkers = false; + } else { + this.webWorkers = true; + console.error("Web Workers is supported in this browser."); + this.frameWorkerManager = new FrameWorkerManager( + width, + height, + "image/png", + ); + } + + this.webWorkers = false; + + this.imageType = "image/jpeg"; + + if (this.webWorkers == false) { + this.zipFileWriter = new BlobWriter(this.imageType); + this.zipWriter = new ZipWriter(this.zipFileWriter); + } + + this.numberOfFrames = 0; + } + + async retrieveFrame( + rawRenderer: THREE.WebGLRenderer, + imageFormat: ImageFormat, + ): Promise { + return new Promise((resolve, reject) => { + rawRenderer.domElement.toBlob( + (blob) => { + if (blob) { + const reader = new FileReader(); + reader.onloadend = () => { + //const imgData = reader.result as string; + // if needed as a base 64 + resolve(blob); + }; + reader.onerror = () => { + reject({ + message: "Failed to Retrieve Frame and Serialize it.", + code: 100, + }); + }; + reader.readAsDataURL(blob); + } + }, + imageFormat, + 1.0, + ); + }); + } + + async setRenderSurfaceSize(width: number, height: number) { + if (this.webWorkers && this.frameWorkerManager) { + this.frameWorkerManager.setCanvas(width, height); + } + } + + async enqueueWork( + rawRenderer: THREE.WebGLRenderer, + scene: THREE.Scene, + camera: THREE.PerspectiveCamera, + ): Promise { + this.numberOfFrames += 1; + if (this.webWorkers && this.frameWorkerManager) { + this.frameWorkerManager.render(scene, camera, this.numberOfFrames); + } else { + const colorRenderTask = this.retrieveFrame(rawRenderer, ImageFormat.JPEG); + this.colorRenderQueue.push(colorRenderTask); + } + } + + private downloadBlob(blob: Blob, filename: string): void { + // Create an object URL for the Blob + const url = URL.createObjectURL(blob); + + // Create a temporary anchor element + const a = document.createElement("a"); + a.href = url; + a.download = filename; // Specify the filename + + // Append the anchor to the document body + document.body.appendChild(a); + + // Trigger a click event on the anchor + a.click(); + + // Clean up by revoking the object URL and removing the anchor element + URL.revokeObjectURL(url); + document.body.removeChild(a); + } + + async equeueAudio(audio: Blob) {} + + async collectColorFrames(final_length: number): Promise { + try { + // webworker requires the COEP. + if (this.webWorkers) { + console.time("ColorQueueRenderTime Worker Time"); + if (this.webWorkers && this.frameWorkerManager) { + console.log(`Collected Number of Frames: ${final_length}`); + // this is sent off to video generation ... + this.payloadZip = + await this.frameWorkerManager.collectedFrames(final_length); + } + console.timeEnd("ColorQueueRenderTime Worker Time"); + } else { + console.time("ColorQueueRenderTime"); + try { + // going to have to zip here with the audio + if (this.webWorkers == false) { + console.log("Using Other Pathway."); + this.colorFrames = await Promise.all(this.colorRenderQueue); + console.log(this.colorFrames); + // Check if zipFileWriter or zipWriter is undefined or null + if (!this.zipFileWriter || !this.zipWriter) { + console.log("Failed to Startup Zip File Writer"); + return; + } + + this.colorFrames.forEach(async (blob, idx) => { + try { + if (!blob || !this.zipFileWriter) { + console.log("Failed to get Blob and Zip File Writer"); + return; + } + + //console.log(`Zipping Frame ${idx} to Zip`); + const name = String(idx).padStart(5, "0"); // '0009' + if (this.zipWriter) { + await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob)); + } else { + console.error("zipWriter is undefined"); + } + } catch (error) { + console.log(error); + } + }); + + this.payloadZip = await this.zipWriter.close(); + //this.downloadBlob(this.payloadZip, "test.zip"); + } + } catch (error) { + console.log("Error in Collecting Color Frames", error); + } + console.timeEnd("ColorQueueRenderTime"); + } + } catch (error) { + console.log(error); + } + + this.hasCache = false; + } + + // Testing function + public downloadImageBlob(blob: Blob, filename: string) { + // Create a URL for the blob + const url = URL.createObjectURL(blob); + + // Create a temporary anchor element + const a = document.createElement("a"); + a.href = url; + a.download = filename; + + // Append the anchor to the document body + document.body.appendChild(a); + + // Trigger a click event on the anchor + a.click(); + + // Remove the anchor from the document + document.body.removeChild(a); + + // Revoke the object URL to free up memory + URL.revokeObjectURL(url); + } + + async clearBuffer(name: BufferType): Promise { + switch (name) { + case BufferType.COLOR: + this.colorFrames = []; + this.colorRenderQueue = []; + this.payloadZip = null; + this.numberOfFrames = 0; + + // reset the writers + this.zipFileWriter = new BlobWriter(this.imageType); + this.zipWriter = new ZipWriter(this.zipFileWriter); + + break; + default: + throw new Error("Invalid buffer name"); + } + } + + async countNonNullFrames(buffer: BufferType): Promise { + return this[buffer].filter((frame) => frame !== null).length; + } + + async checkIfBufferIsEmpty(buffer: BufferType): Promise { + return this[buffer].every((frame) => frame === null); + } + + async logBufferInfo(buffer: BufferType): Promise { + const nonNullFrameCount = await this.countNonNullFrames(buffer); + console.log( + `Buffer Info - ${buffer}: ${nonNullFrameCount} non-null frames`, + ); + } +} diff --git a/app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker.ts b/app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker.ts new file mode 100644 index 0000000000..9a0d98f937 --- /dev/null +++ b/app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker.ts @@ -0,0 +1,15 @@ +// import { VideoAudioPreProcessor } from "app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts"; + +// const processor = new VideoAudioPreProcessor(); + +// // chrome://inspect/#workers debug shared worker. +// self.addEventListener("connect", function (event: MessageEvent) { +// const port = (event as MessageEvent & { ports: MessagePort[] }).ports[0]; + +// port.addEventListener("message", function (event: MessageEvent) { +// console.log("Message received from main script:", event.data); +// port.postMessage("Hello from the Shared Worker!"); +// }); + +// port.start(); +// }); diff --git a/app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts b/app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts new file mode 100644 index 0000000000..6928881eb1 --- /dev/null +++ b/app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts @@ -0,0 +1,31 @@ +class VideoProcessingSharedWorkerManager { + constructor() { + if (typeof SharedWorker !== "undefined") { + console.log("Shared Workers are supported."); + // You can safely create a Shared Worker here + const worker = new SharedWorker( + "app/pages/PageEnigma/Editor/VideoProcessor/shared_workers/video_shared_worker_manager.ts", + { type: "module" }, + ); + + // Get the port for communication + const port = worker.port; + + // Listen for messages from the worker + port.addEventListener("message", function (event) { + console.log("Message received from worker:", event.data); + }); + + // Start the port + port.start(); + + // Send a message to the worker + port.postMessage("Hello from the main script!"); + } else { + console.log("Shared Workers are not supported in this browser."); + } + } + async enqueueWork(frames: [string]) {} + + async completedWork() {} +} diff --git a/app/pages/PageEnigma/Editor/video_audio_preprocessor.ts b/app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts similarity index 67% rename from app/pages/PageEnigma/Editor/video_audio_preprocessor.ts rename to app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts index f805896cf3..7d4d1e5dc1 100644 --- a/app/pages/PageEnigma/Editor/video_audio_preprocessor.ts +++ b/app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts @@ -1,8 +1,7 @@ import { MediaFilesApi } from "~/Classes/ApiManager/MediaFilesApi"; import { BucketConfig } from "~/api/BucketConfig"; -import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; -import { ClipUI } from "../datastructures/clips/clip_ui"; -import * as THREE from "three"; +// import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; +import { ClipUI } from "../../datastructures/clips/clip_ui"; export interface VideoPreProcessorError { message: string; @@ -26,13 +25,6 @@ interface VideoPreProcessorOutputProperties { timelineTrackLengthSeconds: number; } -export interface EngineFrameBuffers { - colorFrames: (string | null)[]; - normalFrames: (string | null)[]; - outlineFrames: (string | null)[]; - depthFrames: (string | null)[]; -} - export interface AudioBuffer { buffer: AudioClipInfo[]; } @@ -51,104 +43,66 @@ export enum ImageFormat { // 44099.10498046875 for the image buffer loop and memory pressure. export class VideoAudioPreProcessor { - ffmpeg: FFmpeg; + // ffmpeg: FFmpeg; properties: VideoPreProcessorOutputProperties; onProgress: (response: StreamingProgressResponse) => void; mediaFilesApi: MediaFilesApi; + bucketConfig: BucketConfig; constructor( onProgress: (response: StreamingProgressResponse) => void, properties: VideoPreProcessorOutputProperties = { frameRate: 60, - imageFormat: ImageFormat.JPEG, + imageFormat: ImageFormat.PNG, timelineTrackLengthSeconds: 7, }, ) { - this.ffmpeg = createFFmpeg({ log: false }); + // this.ffmpeg = createFFmpeg({ log: true, logger: (p) => console.log(p) }); this.properties = properties; this.onProgress = onProgress; this.mediaFilesApi = new MediaFilesApi(); + this.bucketConfig = new BucketConfig(); } // You have to call this early on because it can take a while to load from the CDN. async initialize() { - if (this.ffmpeg.isLoaded() == false) { - await this.ffmpeg.load(); - console.log("Loaded FFMPEG"); - } else { - console.log("Error Failed to Load FFMPEG"); - } + // if (this.ffmpeg.isLoaded() == false) { + // await this.ffmpeg.load(); + // console.log("Video Generation: Loaded FFMPEG"); + // } else { + // console.log("Video Generation: Already Loaded"); + // } } // This retrieves by creating dom elements - async retrieveFrame(rawRenderer: THREE.WebGLRenderer): Promise { - return new Promise((resolve, reject) => { - rawRenderer.domElement.toBlob( - (blob) => { - if (blob) { - const reader = new FileReader(); - reader.onloadend = () => { - const imgData = reader.result as string; - resolve(imgData); - }; - reader.onerror = () => { - reject({ - message: "Failed to Retrieve Frame and Serialize it.", - code: 100, - }); - }; - reader.readAsDataURL(blob); - } - }, - this.properties.imageFormat, - 1.0, - ); - }); + + async removeInMemoryWavs(): Promise { + try { + // const fileNames = this.ffmpeg.FS("readdir", "/"); + // const extension = ".wav"; + // for (const fileName of fileNames) { + // if (fileName.endsWith(extension)) { + // await this.ffmpeg.FS("unlink", fileName); + // } + // } + } catch (error) { + const processError: VideoPreProcessorError = { + message: "Failed to delete files from FFmpeg memory store:", + code: 101, + }; + + const errorMessage = + error instanceof Error ? error.message : String(error); + console.error(errorMessage); + + throw processError; + } } - // parallel version - // async retrieveFrame(rawRenderer: THREE.WebGLRenderer): Promise { - // return new Promise((resolve, reject) => { - // // Create an offscreen canvas - // const offscreenCanvas = document.createElement("canvas"); - // offscreenCanvas.width = rawRenderer.domElement.width; - // offscreenCanvas.height = rawRenderer.domElement.height; - // const ctx = offscreenCanvas.getContext("2d"); - - // if (!ctx) { - // reject("Failed to get 2D context"); - // return; - // } - - // // Draw the renderer's canvas content onto the offscreen canvas - // ctx.drawImage(rawRenderer.domElement, 0, 0); - - // // Convert the offscreen canvas to a Blob - // offscreenCanvas.toBlob( - // (blob) => { - // if (blob) { - // const reader = new FileReader(); - // reader.onloadend = () => { - // const imgData = reader.result as string; - // resolve(imgData); - // }; - // reader.onerror = () => { - // reject("Failed to Retrieve Frame"); - // }; - // reader.readAsDataURL(blob); - // } else { - // reject("Failed to convert canvas to Blob"); - // } - // }, - // "image/jpeg", - // 1.0, - // ); - // }); - // } async removeInMemoryImages(): Promise { try { - const fileNames = this.ffmpeg.FS("readdir", "/"); + // const fileNames = this.ffmpeg.FS("readdir", "/"); const imageFormat = this.properties.imageFormat; let extension = ""; if (imageFormat == ImageFormat.JPEG) { @@ -157,11 +111,11 @@ export class VideoAudioPreProcessor { extension = ".png"; } - for (const fileName of fileNames) { - if (fileName.endsWith(extension)) { - await this.ffmpeg.FS("unlink", fileName); - } - } + // for (const fileName of fileNames) { + // if (fileName.endsWith(extension)) { + // await this.ffmpeg.FS("unlink", fileName); + // } + // } } catch (error) { const processError: VideoPreProcessorError = { message: "Failed to delete files from FFmpeg memory store:", @@ -269,12 +223,12 @@ export class VideoAudioPreProcessor { }); if (response.success && response.data) { - this.bucketConfig.isLocalDev = false; // TODO REMOVE - const imageUrl = this.bucketConfig.getGcsUrl( + this.bucketConfig.isLocalDev = false; + const mediaUrl = this.bucketConfig.getGcsUrl( response.data.public_bucket_path, ); - return imageUrl; + return mediaUrl; } else { const processError: VideoPreProcessorError = { message: "Failed to get MediaUrlFrom Token", @@ -363,86 +317,119 @@ export class VideoAudioPreProcessor { ): Promise { const outputName = `${audioOutputFileName}`; - // Create the samples in memory - for (let i = 0; i < audioBuffer.buffer.length; i++) { - const buffer: AudioClipInfo = audioBuffer.buffer[i]; - const urlPath = await this.getMediaUrlFromToken(buffer.media_id); - const audio = await fetchFile(urlPath); + try { + // Create the samples in memory + for (let i = 0; i < audioBuffer.buffer.length; i++) { + const buffer: AudioClipInfo = audioBuffer.buffer[i]; + const urlPath = await this.getMediaUrlFromToken(buffer.media_id); + const audio = await fetchFile(urlPath); + console.log(urlPath); + console.log(audio); - if (audio.length === 0) { - console.log("audio is 0"); + if (audio.length === 0) { + console.log("audio is 0"); - const processError: VideoPreProcessorError = { - message: `Audio length is 0 for ${urlPath}`, - code: 103, - }; - throw processError; + const processError: VideoPreProcessorError = { + message: `Audio length is 0 for ${urlPath}`, + code: 103, + }; + throw processError; + } + + await this.ffmpeg.FS("writeFile", `tmp${i}.wav`, audio); } + console.log("Write the Audio"); - await this.ffmpeg.FS("writeFile", `tmp${i}.wav`, audio); - } - console.log("Write the Audio"); + // Construct the FFmpeg command to mix the WAV files + const inputArgs = []; + const filterComplexArgs = []; - // Construct the FFmpeg command to mix the WAV files - const inputArgs = []; - const filterComplexArgs = []; - - // Add the silent audio file and account for it below. i + 1 - inputArgs.push( - "-f", - "lavfi", - "-t", - "7", - "-i", - "anullsrc=r=44100:cl=stereo", - ); - - // Add the existing audio clips - for (let i = 0; i < audioBuffer.buffer.length; i++) { - const endTime = audioBuffer.buffer[i].length / this.properties.frameRate; // allows trimming from the right - - const offset = audioBuffer.buffer[i].offset / this.properties.frameRate; // where it will end up on the track. - - const startTime = - audioBuffer.buffer[i].offset / this.properties.frameRate; // allows trimming from the left eventually - const end = endTime - startTime; - - inputArgs.push("-i", `tmp${i}.wav`); - // trim the audio to the end time then offset - filterComplexArgs.push( - `[${i + 1}:a]atrim=start=${0}:end=${end},adelay=${offset * 1000}|${offset * 1000}[a${i}];`, + // Add the silent audio file and account for it below. i + 1 + inputArgs.push( + "-f", + "lavfi", + "-t", + "7", + "-i", + "anullsrc=r=44100:cl=stereo", ); - console.log("Write Audio and Offset Position"); + // Add the existing audio clips + for (let i = 0; i < audioBuffer.buffer.length; i++) { + const endTime = + audioBuffer.buffer[i].length / this.properties.frameRate; // allows trimming from the right + + const offset = audioBuffer.buffer[i].offset / this.properties.frameRate; // where it will end up on the track. + + const startTime = + audioBuffer.buffer[i].offset / this.properties.frameRate; // allows trimming from the left eventually + const end = endTime - startTime; + + inputArgs.push("-i", `tmp${i}.wav`); + // trim the audio to the end time then offset + filterComplexArgs.push( + `[${i + 1}:a]atrim=start=${0}:end=${end},adelay=${offset * 1000}|${offset * 1000}[a${i}];`, + ); + + console.log("Write Audio and Offset Position"); + } + + // Add the silent audio track to the filter complex arguments + filterComplexArgs.push("[0:a]"); + + // Add the existing audio tracks to the filter complex arguments + for (let i = 0; i < audioBuffer.buffer.length; i++) { + filterComplexArgs.push(`[a${i}]`); + } + + // Mix the silent audio track with the existing audio tracks + filterComplexArgs.push(`amix=inputs=${audioBuffer.buffer.length + 1}[a]`); + + // Run the FFmpeg command to mix the WAV files + await this.ffmpeg.run( + ...inputArgs, + "-filter_complex", + filterComplexArgs.join(""), + "-map", + "[a]", + `${outputName}.wav`, + ); + + this.debugListFiles(); + + if (includeBlob) { + const blob = await this.outputWavFromMemory(`${outputName}`); + return blob; + } else { + return; + } + } catch (err) { + console.log(err); + throw err; } + } - // Add the silent audio track to the filter complex arguments - filterComplexArgs.push("[0:a]"); + // Testing function + public downloadImageBlob(blob: Blob, filename: string) { + // Create a URL for the blob + const url = URL.createObjectURL(blob); - // Add the existing audio tracks to the filter complex arguments - for (let i = 0; i < audioBuffer.buffer.length; i++) { - filterComplexArgs.push(`[a${i}]`); - } + // Create a temporary anchor element + const a = document.createElement("a"); + a.href = url; + a.download = filename; - // Mix the silent audio track with the existing audio tracks - filterComplexArgs.push(`amix=inputs=${audioBuffer.buffer.length + 1}[a]`); + // Append the anchor to the document body + document.body.appendChild(a); - // Run the FFmpeg command to mix the WAV files - await this.ffmpeg.run( - ...inputArgs, - "-filter_complex", - filterComplexArgs.join(""), - "-map", - "[a]", - `${outputName}.wav`, - ); + // Trigger a click event on the anchor + a.click(); - if (includeBlob) { - const blob = await this.outputWavFromMemory(`${outputName}`); - return blob; - } else { - return; - } + // Remove the anchor from the document + document.body.removeChild(a); + + // Revoke the object URL to free up memory + URL.revokeObjectURL(url); } async processVideoWithBuffer( @@ -460,8 +447,9 @@ export class VideoAudioPreProcessor { } else { extension = ".png"; } - // console.log("Image Files"); - // console.log(`Buffer Lenght ${buffer.length}`); + console.log(`Image Files of extension ${extension}`); + console.log(`Buffer Length ${buffer.length}`); + for (let index = 0; index < buffer.length; index++) { const element = buffer[index]; if (element === null) { @@ -469,6 +457,7 @@ export class VideoAudioPreProcessor { } // this can fail so catch any errors. const image = await fetchFile(element); + await this.ffmpeg.FS( "writeFile", `${imagePrefix}_image${index}${extension}`, @@ -529,6 +518,18 @@ export class VideoAudioPreProcessor { return blob; } + // Grab the file from in memory fs and provide it as a blob + async outputImageFromMemory( + fileName: string, + imageExtension: ImageFormat = ImageFormat.JPEG, + ): Promise { + // Upload individual videos or single color video + const output = this.ffmpeg.FS("readFile", `${fileName}`); + // Create a Blob from the output file for downloading + const blob = new Blob([output.buffer], { type: `${imageExtension}` }); + return blob; + } + async outputWavFromMemory(fileName: string): Promise { // Upload individual videos or single color video const output = this.ffmpeg.FS("readFile", `${fileName}.wav`); @@ -542,8 +543,6 @@ export class VideoAudioPreProcessor { // List all files in the in-memory filesystem const files = this.ffmpeg.FS("readdir", "/"); await this.deleteFilesFromFFmpegMemoryStore(files); - // This is undefined probably never want to call this really lol - // this.ffmpeg.exit(); } async debugListFiles() { diff --git a/app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts b/app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts new file mode 100644 index 0000000000..b96462afbd --- /dev/null +++ b/app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts @@ -0,0 +1,162 @@ +interface Task { + imageBitmap: ImageBitmap; + width: number; + height: number; + id: number; +} + +import { ClipUI } from "~/pages/PageEnigma/datastructures/clips/clip_ui"; +import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js"; + +class FrameProcessorWorker { + private holdingCanvas: OffscreenCanvas; + private ctx: ImageBitmapRenderingContext | null; + public imageType: string; + + private zipFileWriter: BlobWriter; + private zipWriter: ZipWriter; + + private decodeQueue: Task[]; + + public audio_data: ClipUI[]; + + public isProcessing: boolean; + + public final_length: number; + public item_count: number; + constructor() { + this.final_length = 0; + this.holdingCanvas = new OffscreenCanvas(0, 0); + this.ctx = this.holdingCanvas.getContext("bitmaprenderer"); + this.imageType = "image/jpeg"; + + // Zipping + this.decodeQueue = []; + this.zipFileWriter = new BlobWriter(this.imageType); + this.zipWriter = new ZipWriter(this.zipFileWriter); + + this.audio_data = []; + + this.isProcessing = false; + this.item_count = 0; + } + + async startZipProcessQueue(): Promise { + if (this.decodeQueue.length > 0 && this.isProcessing == false) { + this.isProcessing = true; + + const task = this.decodeQueue.shift(); + if (task) { + const { imageBitmap, width, height, id } = task; + // Proceed with processing the task + + //console.log(`Working On an Frame Id ${id}`); + this.holdingCanvas.width = width; + this.holdingCanvas.height = height; + + try { + // Create or reuse the offscreen canvas + this.ctx?.transferFromImageBitmap(imageBitmap); + + // Convert the canvas content to a Blob + const blob = await this.holdingCanvas.convertToBlob({ + quality: 1.0, + type: this.imageType, + }); + + // Send the Blob back to the main thread + //console.log(`Zipping Frame ${id} to Zip`); + const name = String(id).padStart(5, "0"); // '0009' + await this.zipWriter.add(`${name}.jpg`, new BlobReader(blob)); + + this.item_count++; + } catch (error) { + console.error("Error processing image:", error); + self.postMessage({ error: (error as Error).message }); // + } finally { + this.isProcessing = false; + } + } else { + console.log("Queue is empty, no task to process."); + return; + } + } + // have this function loop + setTimeout(this.startZipProcessQueue.bind(this), 0); + } + + async zipFile() { + // ensure that this queue is done before zipping + console.log(`Waiting to Finish Decoding: ${this.decodeQueue.length}`); + console.log(`Decoded Count: ${this.item_count}`); + console.log(`Final Length: ${this.final_length}`); + if (this.item_count === this.final_length) { + console.log("Zipping"); + const result = await this.zipWriter.close(); + self.postMessage({ result: result }); + // reset the item count and final length + this.item_count = 0; + this.final_length = 0; + + // Zipping + this.decodeQueue = []; + this.zipFileWriter = new BlobWriter(this.imageType); + this.zipWriter = new ZipWriter(this.zipFileWriter); + this.audio_data = []; + this.isProcessing = false; + + return; + } else { + setTimeout(this.zipFile.bind(this), 0); + } + } + + public enqueueTask( + imageBitmap: ImageBitmap, + width: number, + height: number, + id: number, + ): void { + this.decodeQueue.push({ imageBitmap, width, height, id }); + } +} + +const runner = new FrameProcessorWorker(); +let start = false; + +self.addEventListener("message", async (event: MessageEvent) => { + const { + type, + imageBitmap, + width, + height, + frame, + audio_data, + zip, + final_length, + } = event.data; + + // console.log( + // `${type}: ${imageBitmap} ${width} ${height} ${frame} ${audio_data} ${zip} ${final_length}`, + // ); + + if (start == false) { + start = true; + runner.imageType = type; + await runner.startZipProcessQueue(); + } + + if (audio_data || zip) { + if (audio_data) { + runner.audio_data = audio_data; + } + if (zip) { + console.log(`Worker Reported Final Length: ${final_length}`); + runner.final_length = final_length; + await runner.zipFile(); + } + return; + } + + runner.enqueueTask(imageBitmap, width, height, frame); +}); diff --git a/app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_worker_manager.ts b/app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_worker_manager.ts new file mode 100644 index 0000000000..0e6b7b5808 --- /dev/null +++ b/app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_worker_manager.ts @@ -0,0 +1,125 @@ +import * as THREE from "three"; +// import { BlobReader, BlobWriter, ZipWriter } from "@zip.js/zip.js"; + +interface WorkerMessageEvent extends MessageEvent { + data: { + result: Blob; + }; +} + +export class FrameWorkerManager { + private worker: Worker; + private offscreenCanvas: OffscreenCanvas; + private offScreenRenderer: THREE.WebGLRenderer; + public type: string; + + private blobResult: Blob | undefined; + + constructor(width: number, height: number, type: string = "image/jpeg") { + this.type = type; + + this.worker = new Worker( + "app/pages/PageEnigma/Editor/VideoProcessor/workers/frame_processor_worker.ts", + { type: "module" }, + ); + this.worker.addEventListener("message", this.handleMessage.bind(this)); + + this.offscreenCanvas = new OffscreenCanvas(width, height); + this.offScreenRenderer = new THREE.WebGLRenderer({ + canvas: this.offscreenCanvas, + }); + } + + setCanvas(width: number, height: number): void { + this.offscreenCanvas.width = width; + this.offscreenCanvas.height = height; + } + + handleMessage(event: WorkerMessageEvent): void { + const { result } = event.data; + console.log(result); + // zip file + this.blobResult = result; + } + + async sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + async collectedFrames(final_length: number): Promise { + // to get the zip file + console.log(`The Last Frame Number is: ${final_length}`); + this.worker.postMessage({ + type: this.type, + imageBitmap: undefined, + width: undefined, + height: undefined, + frame: undefined, + audio_data: undefined, + zip: true, + final_length: final_length, + }); + + while (this.blobResult === undefined) { + await this.sleep(1); + } + + const result = this.blobResult; + this.blobResult = undefined; + + // For testing. + //this.downloadBlob(result, "test.zip"); + + if (!result) { + throw Error("Failed to Compress Payload"); + } + return result; + } + + private downloadBlob(blob: Blob, filename: string): void { + // Create an object URL for the Blob + const url = URL.createObjectURL(blob); + + // Create a temporary anchor element + const a = document.createElement("a"); + a.href = url; + a.download = filename; // Specify the filename + + // Append the anchor to the document body + document.body.appendChild(a); + + // Trigger a click event on the anchor + a.click(); + + // Clean up by revoking the object URL and removing the anchor element + URL.revokeObjectURL(url); + document.body.removeChild(a); + } + + public terminate(): void { + this.worker.terminate(); + console.log("Worker Terminated"); + } + + public render(scene: THREE.Scene, camera: THREE.Camera, frame: number): void { + this.offScreenRenderer.render(scene, camera); + const imageBitmap = this.offscreenCanvas.transferToImageBitmap(); + this.processFrame(imageBitmap, frame); + } + + private processFrame(imageBitmap: ImageBitmap, frame: number): void { + const width = this.offscreenCanvas.width; + const height = this.offscreenCanvas.height; + const type = this.type; + + this.worker.postMessage({ + type, + imageBitmap, + width, + height, + audio_data: undefined, + frame, + zip: false, + }); + } +} diff --git a/app/pages/PageEnigma/Editor/editor.ts b/app/pages/PageEnigma/Editor/editor.ts index 2e341240b3..9bb31999c2 100644 --- a/app/pages/PageEnigma/Editor/editor.ts +++ b/app/pages/PageEnigma/Editor/editor.ts @@ -36,6 +36,7 @@ import { loadingBarIsShowing, signalScene, } from "~/signals"; + import { outlinerState, updateObjectPanel } from "../signals"; import { IGenerationOptions } from "../models/generationOptions"; import { toEngineActions } from "../Queue/toEngineActions"; @@ -45,10 +46,11 @@ import { SceneManager } from "./scene_manager_api"; import { CustomOutlinePass } from "./CustomOutlinePass.js"; import FindSurfaces from "./FindSurfaces.js"; import { - EngineFrameBuffers, - VideoAudioPreProcessor, -} from "./video_audio_preprocessor"; -import { forEach } from "lodash"; + ImageFormat, + //VideoAudioPreProcessor, +} from "./VideoProcessor/video_audio_preprocessor"; +import { BufferType, EngineFrameBuffers } from "./VideoProcessor/engine_buffer"; + import Stats from "three/examples/jsm/libs/stats.module.js"; export type EditorInitializeConfig = { @@ -93,7 +95,7 @@ class Editor { cameraViewControls: FreeCam | undefined; orbitControls: OrbitControls | undefined; locked: boolean; - frame_buffer: string[][] = []; // Contains a single frame with 4 channels or a signle frame with 1 channel color | depth | outline | normal. + render_timer: number; fps_number: number; cap_fps: number; @@ -174,50 +176,29 @@ class Editor { surfaceFinder: FindSurfaces | undefined; // New Rendering Pipeline Engine Work - videoAudioPreProcessor: VideoAudioPreProcessor; - colorRenderQueue: Array | null>; - depthRenderQueue: Array | null>; - normalRenderQueue: Array | null>; - outlineRenderQueue: Array | null>; + //videoAudioPreProcessor: VideoAudioPreProcessor | undefined; + engineFrameBuffers: EngineFrameBuffers; renderIndex: number; // this should be set in the future to extend the lenght of the track for rendering engine globalSetTrackLengthSeconds: number; + // this is to prevent recording processing from happening twice there is an update loop bug at its core. + processingRecording: boolean; + + // this is to catch and ensure that caching doesn't break the app. + // this happens because we can error out during the video generation process and things will cache despite that failing. + processingHasFailed: boolean; stats: Stats; constructor() { + this.processingHasFailed = false; console.log( "If you see this message twice! then it rendered twice, if you see it once it's all good.", ); - + this.can_initialize = true; + this.processingRecording = false; this.stats = new Stats(); - - - // New Rendering Pipeline Engine Work - this.globalSetTrackLengthSeconds = 7; - this.cap_fps = 60; - const frames = this.globalSetTrackLengthSeconds * this.cap_fps; - - this.videoAudioPreProcessor = new VideoAudioPreProcessor(); - this.videoAudioPreProcessor.initialize(); - - // number of frames max in a scene - this.colorRenderQueue = new Array(frames).fill(null); - this.depthRenderQueue = new Array(frames).fill(null); - this.normalRenderQueue = new Array(frames).fill(null); - this.outlineRenderQueue = new Array(frames).fill(null); - this.renderIndex = 0; - - this.engineFrameBuffers = { - colorFrames: new Array(frames).fill(null), - normalFrames: new Array(frames).fill(null), - outlineFrames: new Array(frames).fill(null), - depthFrames: new Array(frames).fill(null), - }; - - // New Rendering Pipeline Engine Work - // TODO: REMOVE LATER WITH BETTER FIX FOR IMPORTING AMMOJS document.body.appendChild( Object.assign(document.createElement("script"), { @@ -225,8 +206,6 @@ class Editor { }), ); - this.can_initialize = false; - this.can_initialize = true; const newElement = document.createElement("div"); newElement.id = "created-one-element"; document.body.appendChild(newElement); @@ -322,6 +301,24 @@ class Editor { this.globalIpAdapterImage = undefined; // used to display when loading in the app. and to serialize to an image token // TODO REMOVE this.outliner_feature_flag = true; + + // New Rendering Pipeline Engine Work + this.globalSetTrackLengthSeconds = 7; + + // set image type at this stage + + const imageFormat = ImageFormat.JPEG; + + this.renderIndex = 0; + this.engineFrameBuffers = new EngineFrameBuffers( + this.getRenderDimensions().width, + this.getRenderDimensions().height, + ); + if (this.engineFrameBuffers.frameWorkerManager) { + this.engineFrameBuffers.frameWorkerManager.type = imageFormat; + } + + // New Rendering Pipeline Engine Work } getRenderDimensions() { switch (this.render_camera_aspect_ratio) { @@ -413,6 +410,7 @@ class Editor { data: this.render_camera_aspect_ratio, }); } + initialize({ sceneToken, editorCanvasEl, @@ -426,6 +424,9 @@ class Editor { this.can_initialize = false; + // This is to prevent recording processing from happening twice there is an update loop bug at its core. + this.processingRecording = false; + // Gets the canvas. this.canvReference = editorCanvasEl; this.canvasRenderCamReference = camViewCanvasEl; @@ -473,16 +474,10 @@ class Editor { this.renderer.shadowMap.enabled = true; this.clock = new THREE.Clock(); + // Resizes the renderer. this.renderer.setSize(width, height); this.renderer.setPixelRatio(window.devicePixelRatio); - //document.body.appendChild(this.renderer.domElement) - - // window.addEventListener("resize", this.onWindowResize.bind(this)); - // this.renderer.domElement.addEventListener( - // "resize", - // this.onWindowResize.bind(this), - // ); this._configurePostProcessing(); // Controls and movement. @@ -575,7 +570,7 @@ class Editor { this.getAssetType.bind(this), this.setSelected.bind(this), this.isMovable.bind(this), - this.enable_stats.bind(this) + this.enable_stats.bind(this), ); if (this.outliner_feature_flag) { @@ -638,7 +633,7 @@ class Editor { } public enable_stats() { - document.body.appendChild(this.stats.dom) + document.body.appendChild(this.stats.dom); } public async newScene(sceneTitleInput: string) { @@ -810,16 +805,22 @@ class Editor { this.activeScene.scene, this.render_camera, ); + this.render_composer = new EffectComposer(this.rawRenderer, renderTarget); + this.surfaceFinder = new FindSurfaces(); + this.rawRenderPass = new RenderPass( this.activeScene.scene, this.render_camera, ); + this.render_composer.addPass(this.rawRenderPass); + this.render_composer.addPass(this.customOutlinerPass); this.renderOutputPass = new OutputPass(); + this.render_composer.addPass(this.renderOutputPass); this.setColorMap(); @@ -883,7 +884,6 @@ class Editor { this.composer.addPass(this.outlinePass); this.saoPass = new SAOPass(this.activeScene.scene, this.camera); - this.saoPass.params.saoBias = 4.1; this.saoPass.params.saoIntensity = 1.0; this.saoPass.params.saoScale = 32.0; @@ -955,11 +955,15 @@ class Editor { : 1000; this.rawRenderer.setSize(width, height); + + // ensure that during a resize or change in perspective this will processes correct. + this.engineFrameBuffers.setRenderSurfaceSize(width, height); + this.render_camera.aspect = width / height; } this.render_timer += this.clock.getDelta(); - this.frames += 1; + this.playback_location++; this.utils.removeTransformControls(true); @@ -967,53 +971,29 @@ class Editor { if (this.timeline.is_playing) { this.setColorMap(); this.render_composer?.render(); + if (this.canvReference) { + console.time("Color Frame RenderTime"); - // TODO we can fix this, we need to create a bunch of off screen canvas elements - // then in parallel have them be peeled off of there - const renderTask = this.videoAudioPreProcessor.retrieveFrame( - this.rawRenderer, - ); - // await for now race condition with frame retrival on the buffer. - this.engineFrameBuffers.colorFrames[this.renderIndex] = - await renderTask; + // reset this flag to ensure that a failure isn't cached + this.processingHasFailed = false; + // can't render without this. - if (this.engine_preprocessing) { - // Normals - this.setNormalMap(); - this.render_composer?.render(); + if (!this.render_camera) { + return; + } - const normalRenderTask = this.videoAudioPreProcessor.retrieveFrame( + this.engineFrameBuffers.enqueueWork( this.rawRenderer, + this.activeScene.scene, + this.render_camera, ); - this.engineFrameBuffers.normalFrames[this.renderIndex] = - await normalRenderTask; - - //this.normalRenderQueue[this.renderIndex] = normalRenderTask; - - // Depth - this.setRenderDepth(); - this.render_composer?.render(); - const depthRenderTask = this.videoAudioPreProcessor.retrieveFrame( - this.rawRenderer, - ); - this.engineFrameBuffers.depthFrames[this.renderIndex] = - await depthRenderTask; - // this.depthRenderQueue[this.renderIndex] = depthRenderTask; - - // Outline - this.setOutlineRender(); - this.render_composer?.render(); - const outlineRenderTask = this.videoAudioPreProcessor.retrieveFrame( - this.rawRenderer, - ); - this.engineFrameBuffers.outlineFrames[this.renderIndex] = - await outlineRenderTask; - - //this.outlineRenderQueue[this.renderIndex] = outlineRenderTask; this.renderIndex += 1; + console.log(`Frames Counted: ${this.renderIndex}`); + + console.timeEnd("Color Frame RenderTime"); } else { - this.renderIndex += 1; + console.log("We lost the canvas reference."); } this.render_timer += this.clock.getDelta(); @@ -1021,38 +1001,35 @@ class Editor { if (!this.timeline.is_playing) { this.playback_location = 0; try { - this.renderIndex = 0; - // collect all frames here - console.time("RenderQueueTime"); - //const colors = await Promise.all(this.colorRenderQueue); - console.timeEnd("RenderQueueTime"); - // console.log(`Number of Frames: ${colors.length}`); - - // We are doing it sync now. - // this.engineFrameBuffers.colorFrames = colors; - this.colorRenderQueue.fill(null); - - if (this.engine_preprocessing) { - console.time("EnginePreProcessingTime"); - //const normals = await Promise.all(this.normalRenderQueue); - //this.engineFrameBuffers.normalFrames = normals; - //this.normalRenderQueue.fill(null); - - //const depth = await Promise.all(this.depthRenderQueue); - //this.engineFrameBuffers.depthFrames = depth; - //this.depthRenderQueue.fill(null); - - //const outlines = await Promise.all(this.outlineRenderQueue); - //this.engineFrameBuffers.outlineFrames = outlines; - //this.outlineRenderQueue.fill(null); - console.timeEnd("EnginePreProcessingTime"); + if (this.processingRecording) { + console.log( + "processingRecording is already happening this shouldn't have happened", + ); + return; } + this.processingRecording = true; + // collect all frames here, there is some kind of race condition that is happening + // TODO BUG with the main loop,that sends multiple zip requests. + console.log(`COLLECTING COLOR FRAMES COUNTED: ${this.renderIndex}`); + await this.engineFrameBuffers.collectColorFrames(this.renderIndex); + await this.engineFrameBuffers.logBufferInfo(BufferType.COLOR); + this.renderIndex = 0; + console.time("Stop Playback And Upload Video Time"); await this.stopPlaybackAndUploadVideo(); + console.timeEnd("Stop Playback And Upload Video Time"); + + this.processingRecording = false; + console.log("Processing has ended"); } catch (error) { - console.log(error); + // don't use cache in this case. + this.processingHasFailed = true; + this.engineFrameBuffers.clearBuffer(BufferType.COLOR); + console.log(`Video Generation: ${error}`); } - } + } // End Timeline Playing + + // BRB ^)^ } } @@ -1240,15 +1217,13 @@ class Editor { async generateVideo() { // cannot run this function reliably without ensuring state below doesn't blow everything up. if (await this.checkAndUseCache()) { - console.log("Checking Cache"); + console.log("Generating Video: Checking Cache"); await this.videoGeneration.handleCachedEnqueue(); return; } // some state changes below - console.log("Generating Video", this.frame_buffer); - this.timeline.is_playing = false; this.timeline.scrubber_frame_position = 0; this.timeline.current_time = 0; @@ -1266,7 +1241,6 @@ class Editor { console.log("Running without Cache"); this.togglePlayback(); - this.frame_buffer = []; this.render_timer = 0; this.activeScene.renderMode(this.rendering); this.timeline.scrubber_frame_position = 0; @@ -1283,7 +1257,10 @@ class Editor { // This sadly doesn't cover camera framing style changes, // Reasoning is that there is no defined interface to get changes like that. async checkAndUseCache(): Promise { - if (await this.useCachedMediaTokens()) { + if ( + (await this.useCachedMediaTokens()) && + this.processingHasFailed == false + ) { console.log("Using Cache"); return true; } else { @@ -1425,6 +1402,7 @@ class Editor { setupResizeObserver() { this.containerMayReset(); + if (!this.container) { return; } diff --git a/app/pages/PageEnigma/Editor/preview_video_generation.ts b/app/pages/PageEnigma/Editor/preview_video_generation.ts index e4ef1e0c4b..d16ab9151b 100644 --- a/app/pages/PageEnigma/Editor/preview_video_generation.ts +++ b/app/pages/PageEnigma/Editor/preview_video_generation.ts @@ -1,16 +1,12 @@ import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager"; import { - StandardResponse, ProcessStatus, StreamingProgressResponse, VideoAudioPreProcessor, - EngineFrameBuffers, - AudioBuffer, -} from "./video_audio_preprocessor"; +} from "./VideoProcessor/video_audio_preprocessor"; class PreviewVideoGeneration { - conditionedFrameBuffer: EngineFrameBuffers; mediaUploadAPI: MediaUploadApi; videoAPI: VideoApi; videoAndAudioPreProcessor: VideoAudioPreProcessor; @@ -21,12 +17,6 @@ class PreviewVideoGeneration { onProgress: (response: StreamingProgressResponse) => void, ) { this.onProgress = onProgress; - this.conditionedFrameBuffer = { - colorFrames: [], - normalFrames: [], - outlineFrames: [], - depthFrames: [], - }; this.videoAndAudioPreProcessor = new VideoAudioPreProcessor(onProgress); this.mediaUploadAPI = new MediaUploadApi(); this.videoAPI = new VideoApi(); @@ -45,14 +35,7 @@ class PreviewVideoGeneration { // start polling for percentage change and convert and remap to remainint % } - async clearFrameBuffers() { - this.conditionedFrameBuffer = { - colorFrames: [], - normalFrames: [], - outlineFrames: [], - depthFrames: [], - }; - } + async clearFrameBuffers() {} } export { PreviewVideoGeneration }; diff --git a/app/pages/PageEnigma/Editor/scene.ts b/app/pages/PageEnigma/Editor/scene.ts index fbaad0dbf4..efaf956bda 100644 --- a/app/pages/PageEnigma/Editor/scene.ts +++ b/app/pages/PageEnigma/Editor/scene.ts @@ -9,7 +9,7 @@ import { generateUUID } from "three/src/math/MathUtils.js"; import { LoadingPlaceHolderManager } from "./placeholder_manager"; import { MMDAnimationHelper, Water } from "three/examples/jsm/Addons.js"; import { MediaFileType } from "../enums"; -import {ChromaKeyMaterial} from './chromakey' +import { ChromaKeyMaterial } from "./chromakey"; class Scene { name: string; @@ -152,14 +152,20 @@ class Scene { //await videoElement.play(); this.video_planes.push(videoElement); - texture = new THREE.VideoTexture(videoElement); texture.colorSpace = THREE.SRGBColorSpace; - const image_material = new ChromaKeyMaterial(texture, 0x00ff00, 1920, 1080, 0.159, 0.082, 0.0); + const image_material = new ChromaKeyMaterial( + texture, + 0x00ff00, + 1920, + 1080, + 0.159, + 0.082, + 0.0, + ); obj = new THREE.Mesh(geometry, image_material); obj.userData["media_id"] = name; - } else { const loader = new THREE.TextureLoader(); texture = loader.load(image_token); @@ -371,6 +377,7 @@ class Scene { } } + // TODO: REPLACE async getMediaURL(media_id: string) { //This is for prod when we have the proper info on the url. const api_base_url = environmentVariables.values.BASE_API; @@ -426,7 +433,7 @@ class Scene { } setVisible(object_uuid: string, visible: boolean) { - let object = this.get_object_by_uuid(object_uuid); + const object = this.get_object_by_uuid(object_uuid); if (object) { object.visible = visible; object.userData["visible"] = object.visible; @@ -596,7 +603,7 @@ class Scene { child.userData["specular"] = 0.5; child.userData["locked"] = false; child.userData["media_file_type"] = MediaFileType.GLB; - + child.layers.enable(0); child.layers.enable(1); if (load_version <= 1.0) { @@ -678,13 +685,12 @@ class Scene { media_url: string, progress: (event: ProgressEvent) => void, ): Promise { - // TODO: When converted to ts remove this and make ammo await instead. await this.delay_mmd(100); - console.log('Load MMD') - const scriptModule = document.createElement('script'); - scriptModule.type = 'module'; + console.log("Load MMD"); + const scriptModule = document.createElement("script"); + scriptModule.type = "module"; scriptModule.textContent = ` Ammo().then(function (AmmoLib) { @@ -699,7 +705,8 @@ class Scene { return new Promise((resolve, reject) => { const mmdLoader = new MMDLoader(); mmdLoader.loadWithAnimation( - media_url, ["/resources/pose/Lumine Idle cycle.vmd"], + media_url, + ["/resources/pose/Lumine Idle cycle.vmd"], (mmd) => { this.helper.add(mmd.mesh, { animation: mmd.animation, diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index a7b8a9e8e4..e982ff8136 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -240,7 +240,7 @@ export class TimeLine { await this.editorEngine.generateVideo(); this.debounce_generate_video = false; } - console.log("Finished! Generate Video"); + console.log("Generate Video Press Event"); break; } case toEngineActions.CHANGE_CAMERA_ASPECT_RATIO: { @@ -755,9 +755,11 @@ export class TimeLine { // Caps to 10fps so that the buffering issue is solved and it plays i am not sure how to fix this. // TODO: Fix buffering and make 30 fps. const fixedPoint = 1; - + let pb = parseFloat( - (this.scrubber_frame_position / this.editorEngine.cap_fps).toFixed(fixedPoint), + (this.scrubber_frame_position / this.editorEngine.cap_fps).toFixed( + fixedPoint, + ), ); pb = parseFloat((pb % video_plane.duration).toFixed(fixedPoint)); if (video_plane.currentTime !== pb) { @@ -847,9 +849,6 @@ export class TimeLine { } } - - //this.scene.helper.update(0.1); - if ( this.scrubber_frame_position >= this.timeline_limit && this.is_playing diff --git a/app/pages/PageEnigma/Editor/video_generation.ts b/app/pages/PageEnigma/Editor/video_generation.ts index 0a1150183b..d373adc6d1 100644 --- a/app/pages/PageEnigma/Editor/video_generation.ts +++ b/app/pages/PageEnigma/Editor/video_generation.ts @@ -5,8 +5,7 @@ import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; import { ToastTypes, ClipType } from "~/enums"; -import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; -import { ClipUI } from "../datastructures/clips/clip_ui.js"; + import { Visibility } from "./api_manager.js"; import * as THREE from "three"; import { getSceneSignals, addToast, startPollingActiveJobs } from "~/signals"; @@ -14,15 +13,15 @@ import { v4 as uuidv4 } from "uuid"; import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager"; import { globalIPAMediaToken } from "../signals"; -import { VideoPreProcessorError } from "./video_audio_preprocessor"; +import { BufferType } from "./VideoProcessor/engine_buffer"; -// TODO THIS CLASS MAKES NO SENSE Refactor so we generate all the frames first. then pass it through this pipeline as a data structure process it. through this class. +// TODO THIS CLASS MAKES NO SENSE +// Refactor so we generate all the frames first. +// then pass it through this pipeline as a data structure process it. +// through this class. interface MediaTokens { color: string; - normal: string; - depth: string; - outline: string; } export class VideoGeneration { @@ -42,7 +41,7 @@ export class VideoGeneration { this.mediaUploadAPI = new MediaUploadApi(); this.videoAPI = new VideoApi(); this.last_scene_check_sum = ""; - this.last_media_tokens = { color: "", normal: "", depth: "", outline: "" }; + this.last_media_tokens = { color: "" }; this.last_position_of_preprocessing = false; } @@ -50,6 +49,54 @@ export class VideoGeneration { return new Promise((resolve) => setTimeout(resolve, ms)); } + async videoProcessingPipeline(title: string): Promise { + const mediaTokens: MediaTokens = { + color: "", + }; + try { + // TODO send the audio a later time. + // const audioClips = this.editor.timeline.timeline_items.filter((clip) => { + // return clip.type == ClipType.AUDIO; + // }); + + if (!this.editor.engineFrameBuffers.payloadZip) { + await this.handleError( + `Video Processing Failed Please Try Again (zip process failed)`, + 3000, + ); + } + + const zipBlob = this.editor.engineFrameBuffers.payloadZip; + if (!zipBlob) { + await this.handleError(`"Payload blob is null."`, 3000); + } + + const video_upload_response = await this.mediaUploadAPI.UploadStudioShot({ + maybe_title: title, + uuid_idempotency_token: uuidv4(), + blob: zipBlob, + fileName: `${title}.zip`, + maybe_visibility: Visibility.Public, + }); + + if (!video_upload_response.data) { + console.log("Missing media_tokens.color Token"); + const error = { + message: "Missing Data From Video Upload Response", + code: 1, + }; + throw error; + } + + mediaTokens.color = video_upload_response.data; + console.log(`https://storyteller.ai/media_files/${mediaTokens.color}`); + } finally { + await this.editor.engineFrameBuffers.clearBuffer(BufferType.COLOR); + } + + return mediaTokens; + } + // this is to generate previews async generateFrame() { if (!this.editor.generating_preview && this.editor.rawRenderer) { @@ -62,11 +109,6 @@ export class VideoGeneration { }); } - // if (this.editor.render_camera && this.editor.cam_obj) { - // this.editor.render_camera.position.copy(this.editor.cam_obj.position); - // this.editor.render_camera.rotation.copy(this.editor.cam_obj.rotation); - // } - previewSrc.value = ""; this.editor.rawRenderer.setSize( @@ -145,13 +187,6 @@ export class VideoGeneration { // precondition checks, if we have no frames then we shouldn't try to do generations or snapshots // This means no frames so error out - if (this.editor.engineFrameBuffers.colorFrames.length <= 0) { - await this.handleError( - "Failed to Render Nothing to Animate in the Scene.", - 3000, - ); - return; - } this.editor.generating_preview = true; @@ -161,11 +196,8 @@ export class VideoGeneration { "Please stay on this screen and do not switch tabs! while your video is being processed.", }); - const media_tokens: MediaTokens = { + let media_tokens: MediaTokens = { color: "", - normal: "", - depth: "", - outline: "", }; // properties used by the external scope to the for loop. @@ -266,201 +298,15 @@ export class VideoGeneration { message: "Please stay on this screen and do not switch tabs! while your video is being processed.", }); - + // write the end point to upload the video + // upload the color frames with end point. try { - // Sound is only on the color not the other maps - const colorBlob = - await this.editor.videoAudioPreProcessor.processVideoWithBuffer( - this.editor.engineFrameBuffers.colorFrames, - "color", - "color", - "colorFrame", - ); - - await this.editor.videoAudioPreProcessor.removeInMemoryImages(); - //await this.editor.videoAudioPreProcessor.debugListFiles(); - - let shouldProcessAudio = false; - const audioClips = this.editor.timeline.timeline_items.filter((clip) => { - return clip.type == ClipType.AUDIO; - }); - // If there are any audio clips continue processing otherwise this whole process of combining the video - if (audioClips.length > 0) { - shouldProcessAudio = true; - } - - if (shouldProcessAudio) { - const audioBuffer = - await this.editor.videoAudioPreProcessor.audioClipsToAudioBuffer( - audioClips, - ); - - await this.editor.videoAudioPreProcessor.processAudioWithBuffer( - audioBuffer, - "wav_track", - false, - ); - - // debug - //await this.editor.videoAudioPreProcessor.outputWav("wav_track"); - const combinedVideoBlob = - await this.editor.videoAudioPreProcessor.combineAudioAndVideo( - "wav_track", - "color", - "output", // in memory output - ); - - // debug - //await this.editor.videoAudioPreProcessor.outputMp4("output"); - - const video_upload_response = await this.mediaUploadAPI.UploadNewVideo({ - blob: combinedVideoBlob, - fileName: `${title}.mp4`, - uuid: uuidv4(), - maybe_title: title, - maybe_visibility: Visibility.Public, - maybe_style_name: style_name, - maybe_scene_source_media_file_token: immutable_media_token, - }); - - if (!video_upload_response.data) { - console.log("Missing media_tokens.color Token"); - await this.handleError("Failed to Process to Server", 3000); - return; - } - - media_tokens.color = video_upload_response.data; - console.log(`https://storyteller.ai/media/${media_tokens.color}`); - console.log("Uploaded with Audio"); - } else { - // no audio - const video_upload_response = await this.mediaUploadAPI.UploadNewVideo({ - blob: colorBlob, - fileName: `${title}.mp4`, - uuid: uuidv4(), - maybe_title: title, - maybe_visibility: Visibility.Public, - maybe_style_name: style_name, - maybe_scene_source_media_file_token: immutable_media_token, - }); - - if (!video_upload_response.data) { - await this.handleError("Failed to Process to Server", 3000); - return; - } - - media_tokens.color = video_upload_response.data; - console.log(`https://storyteller.ai/media/${media_tokens.color}`); - console.log("Uploaded No Audio"); - } + media_tokens = await this.videoProcessingPipeline(title); } catch (error) { - const err = error as VideoPreProcessorError; - console.log(error); - await this.handleError( - `Failed PreProcess Operation: ${err.message}`, - 3000, - ); - } finally { - // Remove all images in memory for color - await this.editor.videoAudioPreProcessor.removeInMemoryImages(); - this.editor.engineFrameBuffers.colorFrames = []; // possibly allocate up front + await this.handleError(`Video Processing Failed Please Try Again`, 3000); + throw error; } - if (this.editor.engine_preprocessing) { - try { - const depthBlob = await this.enginePreProcessingStep( - "depth", - this.editor.engineFrameBuffers.depthFrames, - ); - - const depth_video_upload_response = - await this.mediaUploadAPI.UploadNewVideo({ - blob: depthBlob, - fileName: `${title}.mp4`, - uuid: uuidv4(), - maybe_title: title, - maybe_visibility: Visibility.Public, - maybe_style_name: style_name, - maybe_scene_source_media_file_token: immutable_media_token, - }); - - if (!depth_video_upload_response.data) { - this.handleError("Failed to Upload PreProcessing 9D", 3000); - return; - } - media_tokens.depth = depth_video_upload_response.data; - - const outlineBlob = await await this.enginePreProcessingStep( - "outline", - this.editor.engineFrameBuffers.outlineFrames, - ); - - const outline_video_upload_response = - await this.mediaUploadAPI.UploadNewVideo({ - blob: outlineBlob, - fileName: `${title}.mp4`, - uuid: uuidv4(), - maybe_title: title, - maybe_visibility: Visibility.Public, - maybe_style_name: style_name, - maybe_scene_source_media_file_token: immutable_media_token, - }); - - if (!outline_video_upload_response.data) { - this.handleError("Failed to Upload PreProcessing 9O", 3000); - return; - } - media_tokens.outline = outline_video_upload_response.data; - - const normalBlob = await this.enginePreProcessingStep( - "normal", - this.editor.engineFrameBuffers.normalFrames, - ); - - const normal_video_upload_response = - await this.mediaUploadAPI.UploadNewVideo({ - blob: normalBlob, - fileName: `${title}.mp4`, - uuid: uuidv4(), - maybe_title: title, - maybe_visibility: Visibility.Public, - maybe_style_name: style_name, - maybe_scene_source_media_file_token: immutable_media_token, - }); - - if (!normal_video_upload_response.data) { - this.handleError("Failed to Upload PreProcessing 9N", 3000); - return; - } - media_tokens.normal = normal_video_upload_response.data; - } catch (err) { - console.log(err); - console.log("Could Not Engine PreProcess"); - // At any point if it fails we need to clear and clean up tokens - media_tokens.depth = ""; - media_tokens.outline = ""; - media_tokens.outline = ""; - } finally { - this.editor.engineFrameBuffers.normalFrames = []; - this.editor.engineFrameBuffers.outlineFrames = []; - this.editor.engineFrameBuffers.depthFrames = []; - - console.log( - `https://storyteller.ai/media/${media_tokens.normal} Normal`, - ); - - console.log(`https://storyteller.ai/media/${media_tokens.depth} Depth`); - - console.log( - `https://storyteller.ai/media/${media_tokens.outline} Outline`, - ); - - await this.editor.videoAudioPreProcessor.removeInMemoryImages(); - } - } - - await this.editor.videoAudioPreProcessor.removeAllVideos(); - this.editor.onWindowResize(); this.editor.setColorMap(); @@ -475,26 +321,9 @@ export class VideoGeneration { this.last_media_tokens = media_tokens; await this.handleEnqueue(media_tokens, ipa_image_token ?? ""); - await this.EndLoadingState(); } - // returns the media token - async enginePreProcessingStep( - frameType: string, - frameBuffer: (string | null)[], - ): Promise { - const preProcessBlob = - await this.editor.videoAudioPreProcessor.processVideoWithBuffer( - frameBuffer, - `${frameType}`, - `${frameType}`, - `${frameType}Frame`, - ); - await this.editor.videoAudioPreProcessor.removeInMemoryImages(); - return preProcessBlob; - } - async handleCachedEnqueue() { const enqueue_studio_response = await this.videoAPI.EnqueueStudio({ enqueueVideo: { @@ -515,9 +344,9 @@ export class VideoGeneration { use_upscaler: this.editor.generation_options.upscale, uuid_idempotency_token: uuidv4(), global_ipa_media_token: globalIPAMediaToken.value ?? "", - input_depth_file: this.last_media_tokens.depth, - input_normal_file: this.last_media_tokens.normal, - input_outline_file: this.last_media_tokens.outline, + input_depth_file: "", + input_normal_file: "", + input_outline_file: "", creator_set_visibility: Visibility.Public, }, }); @@ -555,9 +384,9 @@ export class VideoGeneration { use_upscaler: this.editor.generation_options.upscale, uuid_idempotency_token: uuidv4(), global_ipa_media_token: ipa_image_token, - input_depth_file: upload_tokens.depth, - input_normal_file: upload_tokens.normal, - input_outline_file: upload_tokens.outline, + input_depth_file: "", + input_normal_file: "", + input_outline_file: "", creator_set_visibility: Visibility.Public, }, }); diff --git a/app/root.tsx b/app/root.tsx index d6268b0ca3..92855994ce 100644 --- a/app/root.tsx +++ b/app/root.tsx @@ -57,6 +57,7 @@ export const links: LinksFunction = () => [ }, ]; +// .env part 2 add to this export async function loader() { const env = { // @ts-expect-error ProvessEnv is correct @@ -83,6 +84,10 @@ export async function loader() { CONTEXT: process.env.CONTEXT || "%CONTEXT%", // @ts-expect-error ProvessEnv is correct DEPLOY_CONTEXT: process.env.DEPLOY_CONTEXT || "%DEPLOY_CONTEXT%", + + // .env part 3 + // @ts-expect-error ProvessEnv is correct + UPLOAD_API_VIDEO: process.env.UPLOAD_API_VIDEO || "%UPLOAD_API_VIDEO%", } as Record; return { ENV: env }; } diff --git a/package-lock.json b/package-lock.json index 6519f6926b..ffc80217d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,9 +7,6 @@ "name": "gottagofast_v0.0.0", "dependencies": { "@awesome.me/kit-fde2be5eb0": "^1.0.7", - "@ffmpeg/core": "0.11.0", - "@ffmpeg/ffmpeg": "0.11.6", - "@ffmpeg/util": "0.12.0", "@fortawesome/fontawesome-svg-core": "^6.5.1", "@fortawesome/free-brands-svg-icons": "^6.5.2", "@fortawesome/pro-solid-svg-icons": "^6.5.1", @@ -26,6 +23,7 @@ "@remix-run/serve": "^2.8.1", "@tweenjs/tween.js": "^18.6.4", "@types/lodash": "^4.17.6", + "@zip.js/zip.js": "^2.7.51", "ammo.js": "^0.0.10", "ccapture.js": "github:aceslowman/ccapture.js#npm-fix", "common": "file:common", @@ -93,7 +91,7 @@ "terser": "^5.30.3", "ts-jest": "^29.1.4", "typescript": "^5.1.6", - "vite": "^5.0.0", + "vite": "^5.4.0", "vite-tsconfig-paths": "^4.2.1" }, "engines": { @@ -2846,38 +2844,6 @@ "integrity": "sha512-cEee/Z+I12mZcFJshKcCqC8tuX5hG3s+d+9nZ3LabqKF1vKdF41B92pJVCBggjAGORAeOzyyDDKrZwIkLffeOQ==", "dev": true }, - "node_modules/@ffmpeg/core": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.11.0.tgz", - "integrity": "sha512-9Tt/+2PMpkGPXUK8n6He9G8Y+qR6qmCPSCw9iEKZxHHOvJ9BE/r0Fccj+YgDZTlyu6rXxc9x6EqCaFBIt7qzjA==" - }, - "node_modules/@ffmpeg/ffmpeg": { - "version": "0.11.6", - "resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.11.6.tgz", - "integrity": "sha512-uN8J8KDjADEavPhNva6tYO9Fj0lWs9z82swF3YXnTxWMBoFLGq3LZ6FLlIldRKEzhOBKnkVfA8UnFJuvGvNxcA==", - "dependencies": { - "is-url": "^1.2.4", - "node-fetch": "^2.6.1", - "regenerator-runtime": "^0.13.7", - "resolve-url": "^0.2.1" - }, - "engines": { - "node": ">=12.16.1" - } - }, - "node_modules/@ffmpeg/ffmpeg/node_modules/regenerator-runtime": { - "version": "0.13.11", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", - "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==" - }, - "node_modules/@ffmpeg/util": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@ffmpeg/util/-/util-0.12.0.tgz", - "integrity": "sha512-8sHCW8H/ngqVhbRvCCX4e4uDNgZVoz8uNRry1zphIyzdX6flfBa2TmVwJ4g9/Qw//eAubEnuHaSxDGWWxcOTjg==", - "engines": { - "node": ">=18.17.0" - } - }, "node_modules/@fortawesome/fontawesome-common-types": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.5.1.tgz", @@ -8112,6 +8078,16 @@ "node": ">=12 <14 || 14.2 - 14.9 || >14.10.0" } }, + "node_modules/@zip.js/zip.js": { + "version": "2.7.51", + "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.7.51.tgz", + "integrity": "sha512-RKHaebzZZgQkUuzb49/qweN69e8Np9AUZ9QygydDIrbG1njypSAKwkeqIVeuf2JVGBDyB7Z9HKvzPgYrSlv9gw==", + "engines": { + "bun": ">=0.7.0", + "deno": ">=1.0.0", + "node": ">=16.5.0" + } + }, "node_modules/@zxing/text-encoding": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", @@ -10592,6 +10568,7 @@ "version": "0.1.13", "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dev": true, "optional": true, "dependencies": { "iconv-lite": "^0.6.2" @@ -10601,6 +10578,7 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, "optional": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -13985,11 +13963,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-url": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", - "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==" - }, "node_modules/is-weakmap": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", @@ -17841,6 +17814,7 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -19004,9 +18978,9 @@ } }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -19093,9 +19067,9 @@ } }, "node_modules/postcss": { - "version": "8.4.38", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", - "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", + "version": "8.4.41", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", + "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", "devOptional": true, "funding": [ { @@ -19113,7 +19087,7 @@ ], "dependencies": { "nanoid": "^3.3.7", - "picocolors": "^1.0.0", + "picocolors": "^1.0.1", "source-map-js": "^1.2.0" }, "engines": { @@ -20722,12 +20696,6 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, - "node_modules/resolve-url": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", - "deprecated": "https://github.com/lydell/resolve-url#deprecated" - }, "node_modules/resolve.exports": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", @@ -22461,7 +22429,8 @@ "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true }, "node_modules/tree-kill": { "version": "1.2.2", @@ -23824,13 +23793,13 @@ } }, "node_modules/vite": { - "version": "5.2.8", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.8.tgz", - "integrity": "sha512-OyZR+c1CE8yeHw5V5t59aXsUPPVTHMDjEZz8MgguLL/Q7NblxhZUlTu9xSPqlsUO/y+X7dlU05jdhvyycD55DA==", + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", + "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==", "devOptional": true, "dependencies": { - "esbuild": "^0.20.1", - "postcss": "^8.4.38", + "esbuild": "^0.21.3", + "postcss": "^8.4.40", "rollup": "^4.13.0" }, "bin": { @@ -23850,6 +23819,7 @@ "less": "*", "lightningcss": "^1.21.0", "sass": "*", + "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" @@ -23867,6 +23837,9 @@ "sass": { "optional": true }, + "sass-embedded": { + "optional": true + }, "stylus": { "optional": true }, @@ -23919,10 +23892,26 @@ } } }, + "node_modules/vite/node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.2.tgz", - "integrity": "sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", "cpu": [ "arm" ], @@ -23936,9 +23925,9 @@ } }, "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz", - "integrity": "sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", "cpu": [ "arm64" ], @@ -23952,9 +23941,9 @@ } }, "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.2.tgz", - "integrity": "sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", "cpu": [ "x64" ], @@ -23968,9 +23957,9 @@ } }, "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz", - "integrity": "sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", "cpu": [ "arm64" ], @@ -23984,9 +23973,9 @@ } }, "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz", - "integrity": "sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", "cpu": [ "x64" ], @@ -24000,9 +23989,9 @@ } }, "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz", - "integrity": "sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", "cpu": [ "arm64" ], @@ -24016,9 +24005,9 @@ } }, "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz", - "integrity": "sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", "cpu": [ "x64" ], @@ -24032,9 +24021,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz", - "integrity": "sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", "cpu": [ "arm" ], @@ -24048,9 +24037,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz", - "integrity": "sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", "cpu": [ "arm64" ], @@ -24064,9 +24053,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz", - "integrity": "sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", "cpu": [ "ia32" ], @@ -24080,9 +24069,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz", - "integrity": "sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", "cpu": [ "loong64" ], @@ -24096,9 +24085,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz", - "integrity": "sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", "cpu": [ "mips64el" ], @@ -24112,9 +24101,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz", - "integrity": "sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", "cpu": [ "ppc64" ], @@ -24128,9 +24117,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz", - "integrity": "sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", "cpu": [ "riscv64" ], @@ -24144,9 +24133,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz", - "integrity": "sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", "cpu": [ "s390x" ], @@ -24160,9 +24149,9 @@ } }, "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz", - "integrity": "sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", "cpu": [ "x64" ], @@ -24176,9 +24165,9 @@ } }, "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz", - "integrity": "sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", "cpu": [ "x64" ], @@ -24192,9 +24181,9 @@ } }, "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz", - "integrity": "sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", "cpu": [ "x64" ], @@ -24208,9 +24197,9 @@ } }, "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz", - "integrity": "sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", "cpu": [ "x64" ], @@ -24224,9 +24213,9 @@ } }, "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz", - "integrity": "sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", "cpu": [ "arm64" ], @@ -24240,9 +24229,9 @@ } }, "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz", - "integrity": "sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", "cpu": [ "ia32" ], @@ -24256,9 +24245,9 @@ } }, "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz", - "integrity": "sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", "cpu": [ "x64" ], @@ -24272,9 +24261,9 @@ } }, "node_modules/vite/node_modules/esbuild": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", - "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==", + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", "devOptional": true, "hasInstallScript": true, "bin": { @@ -24284,29 +24273,29 @@ "node": ">=12" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.20.2", - "@esbuild/android-arm": "0.20.2", - "@esbuild/android-arm64": "0.20.2", - "@esbuild/android-x64": "0.20.2", - "@esbuild/darwin-arm64": "0.20.2", - "@esbuild/darwin-x64": "0.20.2", - "@esbuild/freebsd-arm64": "0.20.2", - "@esbuild/freebsd-x64": "0.20.2", - "@esbuild/linux-arm": "0.20.2", - "@esbuild/linux-arm64": "0.20.2", - "@esbuild/linux-ia32": "0.20.2", - "@esbuild/linux-loong64": "0.20.2", - "@esbuild/linux-mips64el": "0.20.2", - "@esbuild/linux-ppc64": "0.20.2", - "@esbuild/linux-riscv64": "0.20.2", - "@esbuild/linux-s390x": "0.20.2", - "@esbuild/linux-x64": "0.20.2", - "@esbuild/netbsd-x64": "0.20.2", - "@esbuild/openbsd-x64": "0.20.2", - "@esbuild/sunos-x64": "0.20.2", - "@esbuild/win32-arm64": "0.20.2", - "@esbuild/win32-ia32": "0.20.2", - "@esbuild/win32-x64": "0.20.2" + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" } }, "node_modules/walker": { @@ -24373,7 +24362,8 @@ "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true }, "node_modules/webm-writer": { "version": "1.0.0", @@ -24399,6 +24389,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" diff --git a/package.json b/package.json index 55402de1c3..edbaee05b5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "remix vite:build", "dev": "remix vite:dev", "staging-build": "remix vite:build", - "lint": "eslint --ignore-path .gitignore --cache --cache-location ./node_modules/.cache/eslint .", + "lint": "eslint --ignore-path .gitignore --cache --cache-location ./node_modules/.cache/eslint playground/* .", "start": "netlify serve", "typecheck": "tsc", "update:icons": "npm update @awesome.me/kit-fde2be5eb0", @@ -19,9 +19,6 @@ }, "dependencies": { "@awesome.me/kit-fde2be5eb0": "^1.0.7", - "@ffmpeg/core": "0.11.0", - "@ffmpeg/ffmpeg": "0.11.6", - "@ffmpeg/util": "0.12.0", "@fortawesome/fontawesome-svg-core": "^6.5.1", "@fortawesome/free-brands-svg-icons": "^6.5.2", "@fortawesome/pro-solid-svg-icons": "^6.5.1", @@ -38,6 +35,7 @@ "@remix-run/serve": "^2.8.1", "@tweenjs/tween.js": "^18.6.4", "@types/lodash": "^4.17.6", + "@zip.js/zip.js": "^2.7.51", "ammo.js": "^0.0.10", "ccapture.js": "github:aceslowman/ccapture.js#npm-fix", "common": "file:common", @@ -105,7 +103,7 @@ "terser": "^5.30.3", "ts-jest": "^29.1.4", "typescript": "^5.1.6", - "vite": "^5.0.0", + "vite": "^5.4.0", "vite-tsconfig-paths": "^4.2.1" }, "engines": { @@ -113,7 +111,7 @@ }, "version": "", "lint-staged": { - "*.js": "eslint --cache --fix", + "*.js": "eslint --cache --fix --ignore-pattern 'playground/**'", "*.{js,css,md}": "prettier --write" } } diff --git a/playground/3jsDemo/ImageProcessingWorker.js b/playground/3jsDemo/ImageProcessingWorker.js new file mode 100644 index 0000000000..2d0b10b01e --- /dev/null +++ b/playground/3jsDemo/ImageProcessingWorker.js @@ -0,0 +1,76 @@ +import * as THREE from "three"; + +export class FrameCollectorManager { + constructor(width, height, expectedFramesLength = 720) { + this.worker = new Worker("canvas-queued-worker.js"); + this.worker.addEventListener("message", this.handleMessage.bind(this)); + + this.frames = []; + + this.expectedFramesLength = expectedFramesLength; + + this.offscreenCanvas = new OffscreenCanvas(width, height); + this.width = width; + this.height = height; + this.offScreenRenderer = new THREE.WebGLRenderer({ + canvas: this.offscreenCanvas, + }); + } + + setCanvas(width, height) { + this.offscreenCanvas.width = width; + this.offscreenCanvas.height = height; + } + + handleMessage(event) { + console.time("Decode"); + console.log("Message from Decode:", event.data); + const { id, blob } = event.data; + console.log(`${id} Frame id: ${blob}`); + console.timeEnd("Decode"); + this.frames.push(blob); + console.log(this.frames.length); + + // create shared worker + if (this.frames.length === this.expectedFramesLength) { + console.log("Complete"); + } + } + + sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + async collectedFrames() { + while (this.frames.length != this.expectedFramesLength) { + await this.sleep(1); + } + + return this.frames; + } + + terminate() { + this.worker.terminate(); + console.log("Worker Terminated"); + } + + render(scene, camera, frame) { + this.offScreenRenderer.render(scene, camera); + const imageBitmap = this.offscreenCanvas.transferToImageBitmap(); + this.processFrame(imageBitmap, frame); + } + + processFrame(imageBitmap, frame) { + const width = this.width; + const height = this.height; + const action = "process"; + + this.worker.postMessage({ + action, + imageBitmap, + width, + height, + frame, + }); + } +} diff --git a/playground/3jsDemo/canvas-queued-worker.js b/playground/3jsDemo/canvas-queued-worker.js new file mode 100644 index 0000000000..0e418f00dd --- /dev/null +++ b/playground/3jsDemo/canvas-queued-worker.js @@ -0,0 +1,64 @@ +// This is responsible for processing and turning the data into + +class ImageProcessorWorker { + constructor() { + this.queue = []; + this.processing = false; + this.holdingCanvas = new OffscreenCanvas(0, 0); + this.ctx = this.holdingCanvas.getContext("bitmaprenderer"); + this.imageType = "image/png"; + } + + enqueueTask(imageBitmap, width, height, id) { + this.queue.push({ imageBitmap, width, height, id }); + } + + async processQueue() { + // If already processing, return to avoid duplicate execution + console.log(`Current Queue Length ${this.queue.length}`); + if (this.queue.length > 0) { + const { imageBitmap, width, height, id } = this.queue.shift(); + console.log(`Working On an Frame Id ${id}`); + + this.holdingCanvas.width = width; + this.holdingCanvas.height = height; + + try { + // Create or reuse the offscreen canvas + this.ctx.transferFromImageBitmap(imageBitmap); + + // Convert the canvas content to a Blob + const blob = await this.holdingCanvas.convertToBlob({ + quality: 1.0, + type: this.imageType, + }); + + // Send the Blob back to the main thread + self.postMessage({ blob, id }); + } catch (error) { + console.error("Error processing image:", error); + self.postMessage({ error: error.message }); + } + } + + console.log("Processing"); + setTimeout(this.processQueue.bind(this), 0); + } +} + +// Instantiate the ImageProcessorWorker +const runner = new ImageProcessorWorker(); +let start = false; + +self.addEventListener("message", async (event) => { + console.log(event.data); + const { type, imageBitmap, width, height, frame } = event.data; + console.log(`${type}: ${imageBitmap} ${width} ${height} ${frame}`); + + if (start == false) { + start = true; + await runner.processQueue(); + } + + runner.enqueueTask(imageBitmap, width, height, frame); +}); diff --git a/playground/3jsDemo/canvas-worker.js b/playground/3jsDemo/canvas-worker.js new file mode 100644 index 0000000000..d5ece42d12 --- /dev/null +++ b/playground/3jsDemo/canvas-worker.js @@ -0,0 +1,17 @@ +// Listen for messages from the main thread +self.addEventListener("message", async function (event) { + const imageBitmap = event.data.imageBitmap; + const width = event.data.width; + const height = event.data.height; + + const holdingCanvas = new OffscreenCanvas(width, height); + // in the worker use queue and pass the imageBitmap + const ctx = holdingCanvas.getContext("bitmaprenderer"); + ctx.transferFromImageBitmap(imageBitmap); + const blob = await holdingCanvas.convertToBlob({ + quality: 1.0, + type: "image/png", + }); + + self.postMessage(blob); // Send the result back to the main thread +}); diff --git a/playground/3jsDemo/canvasworker.js b/playground/3jsDemo/canvasworker.js new file mode 100644 index 0000000000..843e2ffb4d --- /dev/null +++ b/playground/3jsDemo/canvasworker.js @@ -0,0 +1,89 @@ +class CanvasProcessor { + constructor() { + this.queue = []; + this.totalItems = 0; + this.completedItems = 0; + this.running = true; + } + + // Add a canvas to the processing queue + enqueueCanvas(canvas, id) { + this.queue.push({ canvas, id }); + } + + // Set the total number of items to process + setTotalItems(count) { + this.totalItems = count; + } + + // Start the processing loop + async processQueue() { + while (this.running) { + if (this.queue.length > 0) { + await this.processNext(); + } + + // Check if all items have been processed + if (this.completedItems >= this.totalItems) { + self.postMessage({ command: "done", message: "All items processed." }); + this.running = false; + } + + await this.sleep(100); // Small delay to prevent blocking + } + } + + // Process the next canvas in the queue + async processNext() { + if (this.queue.length > 0) { + const { canvas, id } = this.queue.shift(); + + try { + const blob = await canvas.convertToBlob(); + + this.completedItems++; + const percentageComplete = + (this.completedItems / this.totalItems) * 100; + + self.postMessage({ + id, + blob, + completedItems: this.completedItems, + totalItems: this.totalItems, + percentageComplete, + }); + } catch (error) { + self.postMessage({ id, error: error.message }); + } + } + } + + // Utility to sleep for a specified time + sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} + +// Instantiate the processor +const processor = new CanvasProcessor(); + +// Listen for messages from the main thread +self.addEventListener("message", function (event) { + const data = event.data; + + if (data.totalItems) { + processor.setTotalItems(data.totalItems); + } + + if (data.canvas) { + processor.enqueueCanvas(data.canvas, data.id); + } + + if (data.command === "start") { + processor.processQueue(); + } + + if (data.command === "stop") { + processor.running = false; + } +}); diff --git a/playground/3jsDemo/enginepreprocessing.js b/playground/3jsDemo/enginepreprocessing.js new file mode 100644 index 0000000000..a5d0f5569b --- /dev/null +++ b/playground/3jsDemo/enginepreprocessing.js @@ -0,0 +1,29 @@ +// if (this.engine_preprocessing) { +// // Normals +// this.setNormalMap(); +// this.render_composer?.render(); + +// const normalRenderTask = this.videoAudioPreProcessor.retrieveFrame( +// this.rawRenderer, +// ); +// this.engineFrameBuffers.normalFrames[this.renderIndex] = +// await normalRenderTask; +// // Depth +// this.setRenderDepth(); +// this.render_composer?.render(); +// const depthRenderTask = this.videoAudioPreProcessor.retrieveFrame( +// this.rawRenderer, +// ); +// this.engineFrameBuffers.depthFrames[this.renderIndex] = +// await depthRenderTask; + +// // Outline +// this.setOutlineRender(); +// this.render_composer?.render(); +// const outlineRenderTask = this.videoAudioPreProcessor.retrieveFrame( +// this.rawRenderer, +// ); +// this.engineFrameBuffers.outlineFrames[this.renderIndex] = +// await outlineRenderTask; +// this.renderIndex += 1; +// } else { diff --git a/playground/3jsDemo/ffmpeg-worker.js b/playground/3jsDemo/ffmpeg-worker.js new file mode 100644 index 0000000000..e82ae68746 --- /dev/null +++ b/playground/3jsDemo/ffmpeg-worker.js @@ -0,0 +1,121 @@ +// import { VideoGeneration } from "../../app/pages/PageEnigma/Editor/VideoProcessor/video_audio_preprocessor.ts"; + +//import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg"; +self.importScripts("@ffmpeg/ffmpeg"); +const JPEG = "image/jpeg"; +const PNG = "image/png"; + +// chrome://inspect/#workers debug shared worker. + +async function processVideoWithBuffer( + buffer, + fileName, + outFileName, + imagePrefix, +) { + try { + const imageFormat = this.properties.imageFormat; + let extension = ""; + + if (imageFormat == JPEG) { + extension = ".jpeg"; + } else { + extension = ".png"; + } + // console.log("Image Files"); + // console.log(`Buffer Lenght ${buffer.length}`); + for (let index = 0; index < buffer.length; index++) { + const element = buffer[index]; + if (element === null) { + continue; + } + // this can fail so catch any errors. + const image = await fetchFile(element); + await this.ffmpeg.FS( + "writeFile", + `${imagePrefix}_image${index}${extension}`, + image, + ); + } + + await this.ffmpeg.run( + "-framerate", + "" + this.properties.frameRate, + "-i", + `${imagePrefix}_image%d${extension}`, + `${fileName}.mp4`, + ); + + await this.ffmpeg.run( + "-i", + `${fileName}.mp4`, + "-f", + "lavfi", + "-i", + "anullsrc", // This adds a silent audio track + "-max_muxing_queue_size", + "999999", + "-vf", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE + "select=gte(n\\,1)", // WE REMOVED THE BLACK FRAME IN FFMEPG INSTEAD OF THE ENGINE Applies a video filter to remove the first frame. + // "select=gte(n\\,1),scale=1024:576", + "-c:v", + "libx264", // Specify video codec (optional, but recommended for MP4) + "-c:a", + "aac", // Specify audio codec (optional, but recommended for MP4) + "-shortest", // Ensure output duration matches the shortest stream (video or audio) + "-pix_fmt", + "yuv420p", + "-f", + "mp4", + `${outFileName}.mp4`, + ); + + const blob = await this.outputMp4FromMemory(`${outFileName}`); + return blob; + } catch (error) { + console.error(error); + } +} + +self.addEventListener("connect", function (event) { + const port = event.ports[0]; + + port.addEventListener("message", function (event) { + console.log("Message received from main script:", event.data); + port.postMessage("Hello from the Shared Worker!"); + }); + + port.start(); +}); + +// const videoGeneration = new VideoGeneration(); + +// self.addEventListener("connect", function (event) { +// const port = event.ports[0]; + +// console.log("WERE IN"); +// port.start(); + +// port.addEventListener("message", async function (event) { +// console.log("Message received from main script:", event.data); + +// const { colorFrames } = event.data; +// console.log(event.data); + +// try { +// console.time("Processing Video Started"); +// const colorBlob = await videoGeneration.processVideoWithBuffer( +// colorFrames, +// "color", +// "color", +// "colorFrame", +// ); +// console.log(colorBlob); +// console.time("Processing Video Finished"); +// port.postMessage({ colorBlob }); +// } catch (error) { +// console.log(error); +// port.postMessage({ error: error.message }); +// } +// }); +// }); diff --git a/playground/3jsDemo/index.html b/playground/3jsDemo/index.html new file mode 100644 index 0000000000..00e7427f07 --- /dev/null +++ b/playground/3jsDemo/index.html @@ -0,0 +1,25 @@ + + + + + My first three.js app + + + + + + + + diff --git a/playground/3jsDemo/main.js b/playground/3jsDemo/main.js new file mode 100644 index 0000000000..bcd9e867f0 --- /dev/null +++ b/playground/3jsDemo/main.js @@ -0,0 +1,80 @@ +import * as THREE from "three"; + +import { FrameCollectorManager } from "./ImageProcessingWorker.js"; + +const scene = new THREE.Scene(); +const camera = new THREE.PerspectiveCamera( + 75, + window.innerWidth / window.innerHeight, + 0.1, + 1000, +); + +const renderer = new THREE.WebGLRenderer(); +renderer.setSize(window.innerWidth, window.innerHeight); +renderer.setAnimationLoop(animate); +document.body.appendChild(renderer.domElement); + +const geometry = new THREE.BoxGeometry(1, 1, 1); +const material = new THREE.MeshBasicMaterial({ color: 0x00ff00 }); +const cube = new THREE.Mesh(geometry, material); +scene.add(cube); + +camera.position.z = 5; + +const totalFrames = 720; + +const imageProcessing = new FrameCollectorManager( + window.innerWidth, + window.innerHeight, + totalFrames, +); + +let frame = 0; + +// const sharedWorker = new SharedWorker("ffmpeg-worker.js"); +// // Start the port +// sharedWorker.port.start(); +// const message = { message: "Hello" }; +// sharedWorker.port.postMessage(message); + +if (typeof SharedWorker !== "undefined") { + console.log("Shared Workers are supported."); + // You can safely create a Shared Worker here + const worker = new SharedWorker("ffmpeg-worker.js", { type: "module" }); + + // Get the port for communication + const port = worker.port; + + // Listen for messages from the worker + port.addEventListener("message", function (event) { + console.log("Message received from worker:", event.data); + }); + + // Start the port + port.start(); + + // Send a message to the worker + port.postMessage("Hello from the main script!"); +} else { + console.log("Shared Workers are not supported in this browser."); +} + +async function animate() { + cube.rotation.x += 0.01; + cube.rotation.y += 0.01; + + frame += 1; + // buckets the frames. + + if (frame <= totalFrames) { + //console.time("Render"); + // can use the canvas renderer to enqueue as an option. + //imageProcessing.render(scene, camera, frame); + //console.timeEnd("Render"); + } + + // block and wait for all the frames + + renderer.render(scene, camera); +} diff --git a/playground/3jsDemo/run.ps1 b/playground/3jsDemo/run.ps1 new file mode 100644 index 0000000000..b2462b35cd --- /dev/null +++ b/playground/3jsDemo/run.ps1 @@ -0,0 +1,10 @@ +# Run the TypeScript build +npm run build + +# Check if the build was successful +if ($?) { + # Open the index.html file in the default web browser + Start-Process "index.html" +} else { + Write-Host "Build failed. Please check the errors above." +} \ No newline at end of file diff --git a/playground/ts_playground.js b/playground/ts_playground.js new file mode 100644 index 0000000000..63ad388231 --- /dev/null +++ b/playground/ts_playground.js @@ -0,0 +1,35 @@ +class Network { + constructor() {} + + async sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + async task1() { + console.log("World."); + let val = true; + while (val) { + await this.sleep(1000); + console.log("Task1 doing work."); + } + //throw Error("Ded World.") + } + + async task2() { + console.log("Hello."); + let val = true; + while (val) { + await this.sleep(1000); + console.log("Task2 doing work."); + } + //throw Error("Ded Hello.") + } +} + +const network = new Network(); + +async function main() { + network.task1(); + network.task2(); +} + +main(); diff --git a/test/test_bench.js b/test/test_bench.js deleted file mode 100644 index c3c6246cf0..0000000000 --- a/test/test_bench.js +++ /dev/null @@ -1,38 +0,0 @@ - -class Network { - constructor() { } - - async sleep(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); - } - async task1() { - console.log("World.") - let val = true - while (val) { - await this.sleep(1000); - console.log("Task1 doing work.") - } - //throw Error("Ded World.") - } - - async task2() { - console.log("Hello.") - let val = true - while (val) { - await this.sleep(1000); - console.log("Task2 doing work.") - } - //throw Error("Ded Hello.") - } -} - -const network = new Network() - -async function main() { - network.task1() - network.task2() -} - -main() - -