From d834abb64fb1f13096a10a3fe2901e624fe915c3 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 11 Apr 2024 15:56:07 -0400 Subject: [PATCH 1/5] addec debug log for object uuid --- app/pages/PageEnigma/js/editor.ts | 8 +++++--- app/pages/PageEnigma/js/timeline.ts | 6 ++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 805fc5bdff..9392199a59 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -836,9 +836,11 @@ class Editor { async stopPlayback(compile_audio: boolean = true) { - if (this.generating_preview) { - return; - } + console.log(this.frames*(this.timeline.timeline_limit/this.cap_fps)) + + //if (this.generating_preview) { + // return; + //} this.rendering = false; diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 5271a39ef7..4a16b42f55 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -154,6 +154,7 @@ export class TimeLine { obj.userData["name"] = name; obj.name = name; const object_uuid = obj.uuid; + console.log(object_uuid); this.characters[object_uuid] = ClipGroup.CHARACTER; @@ -538,10 +539,7 @@ export class TimeLine { await this.animation_engine.clips[ object.uuid + element.media_id ].play(object); - let fps = 60; - //if (isRendering) { - // fps = 60; - //} + let fps = 120; this.animation_engine.clips[object.uuid + element.media_id].step( this.scrubber_frame_position / fps, // Double FPS for best result. ); From 8ae4c657cf36cc791cb87d26aa5aa64e0728ba7e Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 11 Apr 2024 16:05:11 -0400 Subject: [PATCH 2/5] Added cloning to new item data. --- app/pages/PageEnigma/js/timeline.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 4a16b42f55..4d5bec0307 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -150,6 +150,8 @@ export class TimeLine { const media_id = data.data.media_id; const name = data.data.name; + let new_data = {...data.data}; + const obj = await this.scene.load_glb(media_id); obj.userData["name"] = name; obj.name = name; @@ -158,11 +160,11 @@ export class TimeLine { this.characters[object_uuid] = ClipGroup.CHARACTER; - data.data["object_uuid"] = object_uuid; + new_data["object_uuid"] = object_uuid; Queue.publish({ queueName: QueueNames.FROM_ENGINE, action: fromEngineActions.UPDATE_CHARACTER_ID, - data: data.data, + data: new_data, }); this.addPlayableClip( From 5c8eeb447a52328bd0a50a077abf4232f42d19af Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 11 Apr 2024 16:08:37 -0400 Subject: [PATCH 3/5] fixed fps within stopPlayback --- app/pages/PageEnigma/js/api_manager.ts | 2 +- app/pages/PageEnigma/js/editor.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index 8657109208..7b748060bc 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -191,7 +191,7 @@ export class APIManager { * @param media_file_token * @returns */ - private async getMediaFile(media_file_token: string): Promise { + public async getMediaFile(media_file_token: string): Promise { const api_base_url = "https://api.fakeyou.com"; const url = `${api_base_url}/v1/media_files/file/${media_file_token}`; const response = await fetch(url); diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 9392199a59..4938560281 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -22,7 +22,7 @@ import { AnimationEngine } from "./animation_engine.js"; import { PointerLockControls } from "three/addons/controls/PointerLockControls.js"; import { APPUI_ACTION_TYPES } from "../../../reducers"; -import { ClipGroup } from "~/pages/PageEnigma/models/track"; +import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; import { XYZ } from "../datastructures/common"; import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene"; @@ -836,7 +836,7 @@ class Editor { async stopPlayback(compile_audio: boolean = true) { - console.log(this.frames*(this.timeline.timeline_limit/this.cap_fps)) + let video_fps = this.frames*(this.cap_fps/this.timeline.timeline_limit) //if (this.generating_preview) { // return; @@ -857,7 +857,7 @@ class Editor { } await ffmpeg.run( "-framerate", - "" + this.cap_fps / 2, + "" + video_fps, "-i", "image%d.png", "-f", @@ -882,7 +882,7 @@ class Editor { if (compile_audio) { for (const clip of this.timeline.timeline_items) { - if (clip.type == "lipsync" || clip.type == "audio") { + if (clip.type == ClipType.AUDIO) { await this.convertAudioClip(itteration, ffmpeg, clip); itteration += 1; } From 9a19dcb1b335a9e3b4b4b09b61a84b86297ccedb Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 11 Apr 2024 16:20:48 -0400 Subject: [PATCH 4/5] changed rendering dom element to the preview element. --- app/pages/PageEnigma/js/editor.ts | 22 +++++++--------------- app/pages/PageEnigma/js/timeline.ts | 1 - 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 4938560281..6dc1bc629e 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -666,17 +666,19 @@ class Editor { if (this.composer != null && !this.rendering && this.rawRenderer) { this.composer.render(); this.rawRenderer.render(this.activeScene.scene, this.render_camera); - } else if (this.renderer && this.render_camera) { + } else if (this.renderer && this.render_camera && !this.rendering) { this.renderer.setSize(this.render_width, this.render_height); this.renderer.render(this.activeScene.scene, this.render_camera); + } else if (this.rendering && this.rawRenderer) { + this.rawRenderer.render(this.activeScene.scene, this.render_camera); } else { console.error("Could not render to canvas no render or composer!"); } - if (this.rendering && this.renderer && this.clock) { + if (this.rendering && this.rawRenderer && this.clock) { this.frames += 1; this.playback_location++; - const imgData = this.renderer.domElement.toDataURL(); + const imgData = this.rawRenderer.domElement.toDataURL(); this.frame_buffer.push(imgData); this.render_timer += this.clock.getDelta(); if (this.timeline.is_playing == false) { @@ -835,15 +837,8 @@ class Editor { } async stopPlayback(compile_audio: boolean = true) { - - let video_fps = this.frames*(this.cap_fps/this.timeline.timeline_limit) - - //if (this.generating_preview) { - // return; - //} - + let video_fps = Math.floor(this.frames * (this.cap_fps / this.timeline.timeline_limit)); this.rendering = false; - this.generating_preview = true; const ffmpeg = createFFmpeg({ log: true }); await ffmpeg.load(); @@ -898,15 +893,12 @@ class Editor { const blob = new Blob([output.buffer], { type: "video/mp4" }); const data: any = await this.api_manager.uploadMedia(blob, "render.mp4"); - console.log("data", data); if (data == null) { return; } const upload_token = data["media_file_token"]; console.log(upload_token); - // Create a link to download the file stylize video using api .. - //{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload const result = await this.api_manager .stylizeVideo( @@ -1022,7 +1014,7 @@ class Editor { // This initializes the generation of a video render scene is where the core work happens generateVideo() { console.log("Generating video...", this.frame_buffer); - if (this.rendering || this.generating_preview) { + if (this.rendering) { return; } this.rendering = true; // has to go first to debounce diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 4d5bec0307..646208aa04 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -156,7 +156,6 @@ export class TimeLine { obj.userData["name"] = name; obj.name = name; const object_uuid = obj.uuid; - console.log(object_uuid); this.characters[object_uuid] = ClipGroup.CHARACTER; From e6dbe0116770ab9d4bfaaaadd653f2dd5b8c9f8a Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 11 Apr 2024 17:23:17 -0400 Subject: [PATCH 5/5] fixed audio and implemented smarter recording system using the MediaRecorder API built into the browser. --- app/pages/PageEnigma/js/editor.ts | 82 ++++++++++++++++++++--------- app/pages/PageEnigma/js/timeline.ts | 2 +- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 6dc1bc629e..d0842bf5ea 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -101,6 +101,8 @@ class Editor { art_style: ArtStyle; last_scrub: number; + record_stream: any | undefined; + recorder: MediaRecorder | undefined; // Default params. // scene proxy for serialization @@ -676,15 +678,26 @@ class Editor { } if (this.rendering && this.rawRenderer && this.clock) { - this.frames += 1; - this.playback_location++; - const imgData = this.rawRenderer.domElement.toDataURL(); - this.frame_buffer.push(imgData); + if (this.recorder == undefined) { + this.record_stream = this.rawRenderer.domElement.captureStream(60); // Capture at 30 FPS + this.recorder = new MediaRecorder(this.record_stream, { mimeType: 'video/webm' }); + this.recorder.ondataavailable = (event) => { + if (event.data.size > 0) { + this.frame_buffer.push(event.data); + this.frames += 1; + this.playback_location++; + } + }; + this.recorder.onstop = () => { + this.stopPlayback(); + } + this.recorder.start(); + } + this.render_timer += this.clock.getDelta(); if (this.timeline.is_playing == false) { - this.stopPlayback(); + this.recorder.stop(); this.playback_location = 0; - this.onWindowResize(); } } } @@ -737,7 +750,7 @@ class Editor { } if (this.timeline.is_playing) { - const changeView = await this.timeline.update(this.rendering); + const changeView = await this.timeline.update(delta_time, this.rendering); if (changeView) { this.switchCameraView(); } @@ -837,24 +850,46 @@ class Editor { } async stopPlayback(compile_audio: boolean = true) { - let video_fps = Math.floor(this.frames * (this.cap_fps / this.timeline.timeline_limit)); + //let video_fps = Math.floor(this.frames * (this.cap_fps / this.timeline.timeline_limit)); + //console.log("Video FPS:", video_fps) this.rendering = false; + + console.log(this.frame_buffer); + + + const videoBlob = new Blob(this.frame_buffer, { type: 'video/webm' }); + const videoURL = URL.createObjectURL(videoBlob); + //const arrayBuffer = await videoBlob.arrayBuffer(); + //const uint8Array = new Uint8Array(arrayBuffer); + + // Create an anchor element + let a = document.createElement('a'); + a.href = videoURL; + a.download = 'video.mp4'; // Name of the downloaded file + document.body.appendChild(a); + a.click(); // Trigger the download + + this.generating_preview = true; const ffmpeg = createFFmpeg({ log: true }); await ffmpeg.load(); - for (let index = 0; index < this.frame_buffer.length; index++) { - const element = this.frame_buffer[index]; - await ffmpeg.FS( - "writeFile", - `image${index}.png`, - await fetchFile(element), - ); - } + + //for (let index = 0; index < this.frame_buffer.length; index++) { + // const element = this.frame_buffer[index]; + // await ffmpeg.FS( + // "writeFile", + // `image${index}.jpg`, + // await fetchFile(element), + // ); + //} + + + // Write the Uint8Array to the FFmpeg file system + ffmpeg.FS('writeFile', 'input.webm', await fetchFile(videoURL)); + await ffmpeg.run( - "-framerate", - "" + video_fps, "-i", - "image%d.png", + "input.webm", "-f", "lavfi", "-i", @@ -915,6 +950,7 @@ class Editor { // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} console.log(result); + this.recorder = undefined; } switchPreview() { @@ -1006,8 +1042,6 @@ class Editor { return new Promise((resolve, reject) => { resolve(url); }); - - this.generating_preview = false; } } @@ -1032,14 +1066,14 @@ class Editor { startPlayback() { this.timeline.is_playing = true; this.timeline.scrubber_frame_position = 0; + if (!this.camera_person_mode) { + this.switchCameraView(); + } if (this.activeScene.hot_items) { this.activeScene.hot_items.forEach((element) => { element.visible = false; }); } - if (!this.camera_person_mode) { - this.switchCameraView(); - } } updateSelectedUI() { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 646208aa04..619d0a31c1 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -471,7 +471,7 @@ export class TimeLine { } // called by the editor update loop on each frame - public async update(isRendering = false): Promise { + public async update(delta: number, isRendering = false): Promise { //if (this.is_playing === false) return; // start and stop this.timeline_limit = this.getEndPoint(); if (this.is_playing) {