From 6f7d4dcba9a01134e102861579c982128490b4ae Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 03:45:35 -0400 Subject: [PATCH 1/7] added mute and unmute. Ui does not update but I think its because Chris is not done with the button yet. --- .../datastructures/clips/clip_ui.ts | 2 ++ app/pages/PageEnigma/js/timeline.ts | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts index 04dce4428c..4802143c6f 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts @@ -11,6 +11,7 @@ export class ClipUI { object_uuid: string; offset: number; // in frames length: number; // in frames + should_play: boolean; constructor( version: number, @@ -32,6 +33,7 @@ export class ClipUI { this.media_id = media_id; // Animation / Audio / Lipsync / : Engine this.offset = offset; this.length = length; + this.should_play = true; } toJSON(): string { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 29b801af67..ce5eee18b0 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -124,10 +124,13 @@ export class TimeLine { await this.scrub(data); break; case toEngineActions.MUTE: - await this.mute(data); + await this.mute(data, false); + break; + case toEngineActions.UNMUTE: + await this.mute(data, true); break; default: - console.log("Action Not Wired"); + console.log("Action Not Wired", action); } } @@ -270,8 +273,13 @@ export class TimeLine { console.log(data); } - public async mute(data: any) { - console.log(data); + public async mute(data: any, isMute:boolean) { + this.timeline_items.forEach(element => { + if (element.group == data.data['group']) { + element.should_play = isMute; + console.log(element); + } + }); } public async addPlayableClip(clip: ClipUI): Promise { @@ -368,7 +376,8 @@ export class TimeLine { for (const element of this.timeline_items) { if ( element.offset <= this.scrubber_frame_position && - this.scrubber_frame_position <= element.length + this.scrubber_frame_position <= element.length && + element.should_play == true ) { // run async // element.play() From 2d36849b375e613d4074abe06aa61819031f4d7c Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 03:48:38 -0400 Subject: [PATCH 2/7] Quick fix for characters shrinking to 0, 0, 0 due to react influence. --- app/pages/PageEnigma/js/editor.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index a8f0839986..3157dc7052 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -1211,19 +1211,19 @@ class Editor { version: this.version, objectVectors: { position: { - x: parseFloat(pos.x.toFixed(2)), - y: parseFloat(pos.y.toFixed(2)), - z: parseFloat(pos.z.toFixed(2)), + x: parseFloat(pos.x.toFixed(6)), + y: parseFloat(pos.y.toFixed(6)), + z: parseFloat(pos.z.toFixed(6)), }, rotation: { - x: parseFloat(rot.x.toFixed(2)), - y: parseFloat(rot.y.toFixed(2)), - z: parseFloat(rot.z.toFixed(2)), + x: parseFloat(rot.x.toFixed(6)), + y: parseFloat(rot.y.toFixed(6)), + z: parseFloat(rot.z.toFixed(6)), }, scale: { - x: parseFloat(scale.x.toFixed(2)), - y: parseFloat(scale.y.toFixed(2)), - z: parseFloat(scale.z.toFixed(2)), + x: parseFloat(scale.x.toFixed(6)), + y: parseFloat(scale.y.toFixed(6)), + z: parseFloat(scale.z.toFixed(6)), }, }, }, From d5354e5fbe0905f34b4fd8adc6b5924989d46016 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 03:57:58 -0400 Subject: [PATCH 3/7] Made camera stay selected when in use. --- app/pages/PageEnigma/js/editor.ts | 80 ++++++------------------------- 1 file changed, 14 insertions(+), 66 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 3157dc7052..0c16dc7091 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -574,9 +574,10 @@ class Editor { switchCameraView() { this.camera_person_mode = !this.camera_person_mode; - console.log("camera"); + console.log(this.camera_person_mode); if (this.cam_obj) { if (this.camera_person_mode) { + this.last_cam_pos.copy(this.camera.position); this.last_cam_rot.copy(this.camera.rotation); @@ -597,6 +598,11 @@ class Editor { element.visible = false; }); } + + this.removeTransformControls(); + this.selected = this.cam_obj; + this.dispatchAppUiState({type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT}); + this.updateSelectedUI(); } else { this.camera.position.copy(this.last_cam_pos); this.camera.rotation.copy(this.last_cam_rot); @@ -615,6 +621,10 @@ class Editor { element.visible = true; }); } + + this.dispatchAppUiState({ + type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT, + }); } } } @@ -782,38 +792,6 @@ class Editor { this.activeScene.renderMode(this.rendering); } - save() { - //console.log(this.control) - //if(this.selected != null){ this.control.detach(this.selected) } - //this.activeScene.scene.remove(this.control) - //this.activeScene.scene.remove(this.activeScene.gridHelper) - //this.save_manager.save(this.activeScene.scene, this._save_to_cloud.bind(this), this.audio_manager, this.timeline, this.activeScene.animations) - //this.activeScene._createGrid() - //this.audio_engine.playClip("m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg") - //console.log(this.selected) - - if (this.selected == null) { - return; - } - this.transform_engine.addFrame(this.selected); - console.log("Frame taken."); - } - - take_timeline_cam_clip() { - if (this.cam_obj == null) { - return; - } - if (!this.camera_person_mode) { - return; - } - this.transform_engine.addFrame( - this.cam_obj, - this.transform_engine.clips[this.cam_obj.uuid].length, - ); - console.log("Camera frame taken."); - this.activeScene.createPoint(this.cam_obj.position, false); - } - // Basicly Unity 3D's update loop. async updateLoop(time: number) { if (this.cam_obj == undefined) { @@ -1327,39 +1305,9 @@ class Editor { this.outlinePass.selectedObjects = [this.selected]; this.transform_interaction = true; - // Contact react land - const pos = this.selected.position; - const rot = this.selected.rotation; - const scale = this.selected.scale; - this.dispatchAppUiState({ - type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT, - payload: { - group: - this.selected.name === "::CAM::" - ? ClipGroup.CAMERA - : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips - object_uuid: this.selected.uuid, - object_name: this.selected.name, - version: this.version, - objectVectors: { - position: { - x: parseFloat(pos.x.toFixed(2)), - y: parseFloat(pos.y.toFixed(2)), - z: parseFloat(pos.z.toFixed(2)), - }, - rotation: { - x: parseFloat(rot.x.toFixed(2)), - y: parseFloat(rot.y.toFixed(2)), - z: parseFloat(rot.z.toFixed(2)), - }, - scale: { - x: parseFloat(scale.x.toFixed(2)), - y: parseFloat(scale.y.toFixed(2)), - z: parseFloat(scale.z.toFixed(2)), - }, - }, - }, - }); + // Contact react land + this.dispatchAppUiState({type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT}); + this.updateSelectedUI(); } } else if (this.transform_interaction == false) { this.removeTransformControls(); From 71860db42bbf3f0e6b713b15230219bb8001c3ad Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 03:59:51 -0400 Subject: [PATCH 4/7] code cleanup and quick fix for selection. --- app/pages/PageEnigma/js/editor.ts | 79 +------------------------------ 1 file changed, 2 insertions(+), 77 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 0c16dc7091..0b8490345d 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -629,28 +629,6 @@ class Editor { } } - async addTransformClipBase( - name: string = "New Clip", - group: "object" | "character" | "camera" | "global_audio", - object: THREE.Object3D, - offset: number, - length: number, - ) { - this.timeline.addPlayableClip( - new ClipUI( - 1.0, - "transform", - group, - name, - object.uuid, // object id here ... - object.uuid, // Need to change to media id .... - offset, - length, - ), - ); - this.transform_engine.loadObject(object.uuid, length); - } - async _test_demo() { // note the database from the server is the source of truth for all the data. // Test code here @@ -663,60 +641,6 @@ class Editor { // Stick Open Pose Man: m_9f3d3z94kk6m25zywyz6an3p43fjtw // XBot: m_r7w1tmkx2jg8nznr3hyzj4k6zhfh7d // YBot: m_9sqg0evpr23587jnr8z3zsvav1x077 - - // Load timeline creates the the clips from the datastructure and loads them in here. - // load object into the engine for lip syncing - //this.lipsync_engine.load_object( - // object.uuid, - // "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - //); - //this.animation_engine.load_object( - // object.uuid, - // "/resources/models/fox/fox_idle.glb", - // "clip3", - //); - // then it creates clip ui to load the playable clips - // then refreshes the timeline. - // create the clip with the same id for a reference to the media - //this.timeline.addPlayableClip( - // new ClipUI( - // 1.0, - // "lipsync", - // "character", - // "clip1", - // "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", - // object.uuid, - // 150, - // 400, - // ), - //); - //// media id for this is up in the air but when a path is created you should be able to store and delete it - //this.timeline.addPlayableClip( - // new ClipUI(1.0, "transform", "character", "clip2", object.uuid, object.uuid, 0, 150), - //); - // media id for this as well it can be downloaded - //this.timeline.addPlayableClip( - // new ClipUI( - // 1.0, - // "animation", - // "character", - // "clip3", - // "/resources/models/fox/fox_idle.glb", - // object.uuid, - // 0, - // 400, - // ), - //); - //this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v"); - //await this.timeline.addPlayableClip(new ClipUI( - // 1.0, - // "audio", - // "global_audio", - // "AudioClip", - // "m_h33vytxs5eqqqf07nsy14qzrf9ww4v", - // "", - // 0, - // 50)); } // Configure post processing. @@ -1269,7 +1193,8 @@ class Editor { this.raycaster == undefined || this.mouse == undefined || this.control == undefined || - this.outlinePass == undefined + this.outlinePass == undefined || + this.camera_person_mode ) { return; } From d06a4c7b71185ecd39f4aadfd54e7fb707b83fd8 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 04:07:21 -0400 Subject: [PATCH 5/7] moved some code arround to locations they make sense. --- app/pages/PageEnigma/js/editor.ts | 134 ++++++++++-------------------- 1 file changed, 44 insertions(+), 90 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 0b8490345d..70044bf758 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -292,7 +292,22 @@ class Editor { window.addEventListener("mousemove", this.onMouseMove.bind(this), false); window.addEventListener("click", this.onMouseClick.bind(this), false); // Base control and debug stuff remove debug in prod. - this._initializeControl(); + if (this.control == undefined) { + return; + } + this.control.addEventListener("change", this.renderScene.bind(this)); + this.control.addEventListener("dragging-changed", (event: any) => { + if (this.orbitControls == undefined) { + return; + } + this.orbitControls.enabled = !event.value; + this.updateSelectedUI(); + // this.update_properties() + }); + this.control.setSize(0.5); // Good default value for visuals. + this.raycaster = new THREE.Raycaster(); + this.mouse = new THREE.Vector2(); + this.activeScene.scene.add(this.control); // Resets canvas size. this.onWindowResize(); // Creates the main update loop. @@ -312,32 +327,11 @@ class Editor { this.onMouseUp.bind(this), false, ); - this.renderer.domElement.addEventListener( - "onContextMenu", - this.onContextMenu.bind(this), - false, - ); // saving state of the scene this.current_scene_media_token = null; this.current_scene_glb_media_token = null; - this.renderer.domElement.addEventListener( - "mousedown", - this.onMouseDown.bind(this), - false, - ); - this.renderer.domElement.addEventListener( - "mouseup", - this.onMouseUp.bind(this), - false, - ); - this.renderer.domElement.addEventListener( - "onContextMenu", - this.onContextMenu.bind(this), - false, - ); - this.cam_obj = this.activeScene.get_object_by_name("::CAM::"); this.dispatchAppUiState({ @@ -365,21 +359,6 @@ class Editor { public async testTestTimelineEvents() { } - public async testStylizeRequest() { - const result = await this.api_manager - .stylizeVideo( - "mu_6wy1570a0c3c0tpkkncf4tsvb5234", - this.art_style, - this.positive_prompt, - this.negative_prompt, - Visibility.Public, - ) - .catch((error) => { - console.log(error); - }); - console.log(result); - } - public async loadScene(scene_media_token: string) { this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, @@ -453,7 +432,6 @@ class Editor { } } - public async saveScene(name: string) { // remove controls when saving scene. this.removeTransformControls(); @@ -716,6 +694,34 @@ class Editor { this.activeScene.renderMode(this.rendering); } + // Render the scene to the camera, this is called in the update. + renderScene() { + if (this.composer != null && !this.rendering && this.rawRenderer) { + this.composer.render(); + this.rawRenderer.render(this.activeScene.scene, this.render_camera); + } else if (this.renderer && this.render_camera) { + this.renderer.setSize(this.render_width, this.render_height); + this.renderer.render(this.activeScene.scene, this.render_camera); + } else { + console.error("Could not render to canvas no render or composer!"); + } + + if (this.rendering && this.renderer && this.clock) { + this.frames += 1; + this.playback_location++; + const imgData = this.renderer.domElement.toDataURL(); + this.frame_buffer.push(imgData); + this.render_timer += this.clock.getDelta(); + if (this.timeline.is_playing == false) { + this.stopPlayback(); + this.playback_location = 0; + this.rendering = false; + this.switchCameraView(); + this.onWindowResize(); + } + } + } + // Basicly Unity 3D's update loop. async updateLoop(time: number) { if (this.cam_obj == undefined) { @@ -1043,54 +1049,6 @@ class Editor { } } - // Initializes transform x y z changes. - _initializeControl() { - if (this.control == undefined) { - return; - } - this.control.addEventListener("change", this.renderScene.bind(this)); - this.control.addEventListener("dragging-changed", (event: any) => { - if (this.orbitControls == undefined) { - return; - } - this.orbitControls.enabled = !event.value; - this.updateSelectedUI(); - // this.update_properties() - }); - this.control.setSize(0.5); // Good default value for visuals. - this.raycaster = new THREE.Raycaster(); - this.mouse = new THREE.Vector2(); - this.activeScene.scene.add(this.control); - } - - // Render the scene to the camera, this is called in the update. - renderScene() { - if (this.composer != null && !this.rendering && this.rawRenderer) { - this.composer.render(); - this.rawRenderer.render(this.activeScene.scene, this.render_camera); - } else if (this.renderer && this.render_camera) { - this.renderer.setSize(this.render_width, this.render_height); - this.renderer.render(this.activeScene.scene, this.render_camera); - } else { - console.error("Could not render to canvas no render or composer!"); - } - - if (this.rendering && this.renderer && this.clock) { - this.frames += 1; - this.playback_location++; - const imgData = this.renderer.domElement.toDataURL(); - this.frame_buffer.push(imgData); - this.render_timer += this.clock.getDelta(); - if (this.timeline.is_playing == false) { - this.stopPlayback(); - this.playback_location = 0; - this.rendering = false; - this.switchCameraView(); - this.onWindowResize(); - } - } - } - updateSelectedUI() { if (this.selected == undefined) { @@ -1161,10 +1119,6 @@ class Editor { this.render_camera.updateProjectionMatrix(); } - onContextMenu(event: any) { - return false; - } - onMouseDown(event: any) { if (event.button === 1 && this.camera_person_mode) { this.lockControls?.lock(); From 8e8c2a8bb9526e1d78d77195de7f5c4f3170d168 Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 04:10:02 -0400 Subject: [PATCH 6/7] removed useless function. --- app/pages/PageEnigma/js/editor.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 70044bf758..80b381be29 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -785,12 +785,6 @@ class Editor { this.transform_interaction = true; } - async loadWavAsBlob(url: string) { - const response = await fetch(url); - const blob = await response.blob(); - return blob; - } - async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) { const video_og = itteration + "tmp.mp4"; const wav_name = itteration + "tmp.wav"; From 05060f2abaf58f296579ae10818b75863c97718a Mon Sep 17 00:00:00 2001 From: Quicksticks-oss Date: Sat, 6 Apr 2024 04:17:30 -0400 Subject: [PATCH 7/7] Added step forward and backward. --- .../ControlsTopButtons/ControlsTopButtons.tsx | 8 +++--- .../comps/ControlsVideo/ControlsVideo.tsx | 4 +-- app/pages/PageEnigma/js/editor.ts | 4 +++ app/pages/PageEnigma/js/timeline.ts | 26 ++++++++++++------- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index 527bd5226b..9d2db03975 100644 --- a/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -23,7 +23,7 @@ export const ControlsTopButtons = () => { }; const handleButtonTest = () => { - editorEngine?.testStylizeRequest(); + // editorEngine?.testStylizeRequest(); }; const handleTestButton2 = () => { @@ -37,7 +37,7 @@ export const ControlsTopButtons = () => { editorEngine?.startPlayback(); }; const handleButtonTakeFrame = () => { - editorEngine?.take_timeline_cam_clip(); + // editorEngine?.take_timeline_cam_clip(); }; const handleButtonSingleFrame = () => { editorEngine?.generateFrame(); @@ -112,8 +112,8 @@ export const ControlsTopButtons = () => {
- - + {/* + */}