From 5d0cfddc90de7316b95813c5a5bda3f2b78aea07 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Thu, 4 Apr 2024 23:27:12 -0400 Subject: [PATCH 1/2] =?UTF-8?q?added=20Cartoon,=20J=C3=A9ja=20-=20On=20&?= =?UTF-8?q?=20On=20(feat.=20Daniel=20Levi)=20|=20Electronic=20Pop=20|=20NC?= =?UTF-8?q?S=20-=20Copyright=20Free=20Music?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PageEnigma/contexts/TrackContext/TrackProvider.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index 3c2a7efda1..1dc4c59bb0 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -129,6 +129,13 @@ export const TrackProvider = ({ children }: Props) => { type: ClipType.AUDIO, length: 25, name: "Talk", + }, + { + version: 1, + media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt", + type: ClipType.AUDIO, + length: 25, + name: "NCS Song", } ]); setScale(1); From f06f8cae12a600714acf7b4a9d56e4d947757420 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Fri, 5 Apr 2024 00:11:48 -0400 Subject: [PATCH 2/2] fixed rendering and video uploads. --- app/pages/PageEnigma/js/api_manager.ts | 2 +- app/pages/PageEnigma/js/editor.ts | 134 +++++++++++++------------ app/pages/PageEnigma/js/timeline.ts | 6 +- 3 files changed, 75 insertions(+), 67 deletions(-) diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index fde45ddd37..1a9ad9c151 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -310,7 +310,7 @@ export class APIManager { blob: any, fileName: string, ): Promise { - const url = `${this.baseUrl}/v1/media_uploads/upload`; + const url = `${this.baseUrl}/v1/media_files/upload`; const uuid = uuidv4(); const formData = new FormData(); diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index bf42059add..3e8c1e7736 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -91,6 +91,7 @@ class Editor { cam_obj: THREE.Object3D | undefined; renderPass: RenderPass | undefined; generating_preview: boolean; + frames: number; camera_person_mode: boolean; current_scene_media_token: string | null; @@ -172,6 +173,7 @@ class Editor { this.playback_location = 0; this.max_length = 10; this.last_scrub = 0; + this.frames = 0; // Audio Engine Test. this.render_width = 1280; @@ -361,7 +363,7 @@ class Editor { console.log(result); } - public async testTestTimelineEvents() {} + public async testTestTimelineEvents() { } public async testStylizeRequest() { const result = await this.api_manager @@ -433,8 +435,8 @@ class Editor { } // TO UPDATE selected objects in the scene might want to add to the scene ... - async setSelectedObject(position:XYZ,rotation:XYZ,scale:XYZ) { - if (this.selected !=undefined || this.selected !=null) { + async setSelectedObject(position: XYZ, rotation: XYZ, scale: XYZ) { + if (this.selected != undefined || this.selected != null) { //console.log(`triggering setSelectedObject`) this.selected.position.x = position.x this.selected.position.y = position.y @@ -449,7 +451,7 @@ class Editor { this.selected.scale.z = scale.z } - } + } public async saveScene(name: string) { @@ -813,7 +815,7 @@ class Editor { } // Basicly Unity 3D's update loop. - updateLoop(time: number) { + async updateLoop(time: number) { if (this.cam_obj == undefined) { this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); } @@ -831,11 +833,11 @@ class Editor { if (this.cameraViewControls && this.camera_person_mode) { this.cameraViewControls.update(5 * delta_time); - if(this.cam_obj){ + if (this.cam_obj) { if (this.last_scrub != this.timeline.scrubber_frame_position) { this.camera.position.copy(this.cam_obj.position); this.camera.rotation.copy(this.cam_obj.rotation); - } else if(this.timeline.is_playing == false) { + } else if (this.timeline.is_playing == false) { this.cam_obj.position.copy(this.camera.position); this.cam_obj.rotation.copy(this.camera.rotation); } else { @@ -846,25 +848,24 @@ class Editor { this.cam_obj.visible = false; } } - else if (this.cam_obj){ + else if (this.cam_obj) { this.cam_obj.visible = true; } if (this.render_camera && this.cam_obj) { this.render_camera.position.copy(this.cam_obj.position); this.render_camera.rotation.copy(this.cam_obj.rotation); - this.cam_obj.scale.copy(new THREE.Vector3(1,1,1)); + this.cam_obj.scale.copy(new THREE.Vector3(1, 1, 1)); } - - if(this.timeline.is_playing){ - this.timeline.update(); - } - else if (this.last_scrub == this.timeline.scrubber_frame_position){ + + if (this.timeline.is_playing) { + await this.timeline.update(this.rendering); + } + else if (this.last_scrub == this.timeline.scrubber_frame_position) { this.updateSelectedUI(); } this.last_scrub = this.timeline.scrubber_frame_position; - this.renderScene(); } @@ -933,10 +934,10 @@ class Editor { audioSegment, "-filter_complex", "[1:a]adelay=" + - startTime * 1000 + - "|" + - startTime * 1000 + - "[a1];[0:a][a1]amix=inputs=2[a]", + startTime * 1000 + + "|" + + startTime * 1000 + + "[a1];[0:a][a1]amix=inputs=2[a]", "-map", "[a]", `${itteration}final_tmp.wav`, @@ -962,6 +963,8 @@ class Editor { } async stopPlayback(compile_audio: boolean = true) { + console.log(this.frames, this.frame_buffer.length); + this.renderMode(); const ffmpeg = createFFmpeg({ log: true }); await ffmpeg.load(); @@ -975,7 +978,7 @@ class Editor { } await ffmpeg.run( "-framerate", - "" + this.cap_fps, + "" + this.cap_fps/2, "-i", "image%d.png", "-f", @@ -1020,22 +1023,24 @@ class Editor { console.log("data", data) if (data == null) { return; } - let upload_token = data['upload_token']; + let upload_token = data['media_file_token']; console.log(upload_token); // Create a link to download the file stylize video using api .. //{"success":true,"upload_token":"mu_x9kr5cfafn512pjbygdszvbdpktrr"} payload const result = await this.api_manager - .stylizeVideo( - upload_token, - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - ) - .catch((error) => { - console.log(error); - }); + .stylizeVideo( + upload_token, + this.art_style, + this.positive_prompt, + this.negative_prompt, + Visibility.Public, + ) + .catch((error) => { + console.log(error); + }); + + // {"success":true,"inference_job_token":"jinf_j3nbqbd15wqxb0xcks13qh3f3bz"} console.log(result); @@ -1167,6 +1172,7 @@ class Editor { } if (this.rendering && this.renderer && this.clock) { + this.frames += 1; this.playback_location++; const imgData = this.renderer.domElement.toDataURL(); this.frame_buffer.push(imgData); @@ -1186,41 +1192,41 @@ class Editor { if (this.selected == undefined) { return; } - const pos = this.selected.position; - const rot = this.selected.rotation; - const scale = this.selected.scale; - - // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. - this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, - payload: { - group: - this.selected.name === "::CAM::" - ? ClipGroup.CAMERA - : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips - object_uuid: this.selected.uuid, - object_name: this.selected.name, - version: this.version, - objectVectors: { - position: { - x: parseFloat(pos.x.toFixed(2)), - y: parseFloat(pos.y.toFixed(2)), - z: parseFloat(pos.z.toFixed(2)), - }, - rotation: { - x: parseFloat(rot.x.toFixed(2)), - y: parseFloat(rot.y.toFixed(2)), - z: parseFloat(rot.z.toFixed(2)), - }, - scale: { - x: parseFloat(scale.x.toFixed(2)), - y: parseFloat(scale.y.toFixed(2)), - z: parseFloat(scale.z.toFixed(2)), - }, + const pos = this.selected.position; + const rot = this.selected.rotation; + const scale = this.selected.scale; + + // TODO this is a bug we need to only show when clicked on and use UPDATE when updating. + this.dispatchAppUiState({ + type: APPUI_ACTION_TYPES.UPDATE_CONTROLPANELS_SCENEOBJECT, + payload: { + group: + this.selected.name === "::CAM::" + ? ClipGroup.CAMERA + : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips + object_uuid: this.selected.uuid, + object_name: this.selected.name, + version: this.version, + objectVectors: { + position: { + x: parseFloat(pos.x.toFixed(2)), + y: parseFloat(pos.y.toFixed(2)), + z: parseFloat(pos.z.toFixed(2)), + }, + rotation: { + x: parseFloat(rot.x.toFixed(2)), + y: parseFloat(rot.y.toFixed(2)), + z: parseFloat(rot.z.toFixed(2)), + }, + scale: { + x: parseFloat(scale.x.toFixed(2)), + y: parseFloat(scale.y.toFixed(2)), + z: parseFloat(scale.z.toFixed(2)), }, }, - }); - + }, + }); + } // Automaticly resize scene. diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 093c6f9da0..29b801af67 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -347,7 +347,7 @@ export class TimeLine { } // called by the editor update loop on each frame - public async update() { + public async update(isRendering=false) { //if (this.is_playing == false) return; // start and stop this.timeline_limit = this.getEndPoint(); if (this.is_playing) { @@ -399,8 +399,10 @@ export class TimeLine { } else if (element.type === ClipType.ANIMATION) { if (object) { await this.animation_engine.clips[object.uuid+element.media_id].play(object); + let fps = 120; + if(isRendering) { fps = 60; } this.animation_engine.clips[object.uuid+element.media_id].step( - this.scrubber_frame_position / 120, // Double FPS for best result. + this.scrubber_frame_position / fps, // Double FPS for best result. ); } } else {