diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index c20b871c69..e426334ad3 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -126,7 +126,7 @@ export const TrackProvider = ({ children }: Props) => { }, { version: 1, - media_id: uuid.v4(), + media_id: "m_403phjvjkbbaxxbz8y7r6qjay07mfd", type: ClipType.AUDIO, length: 25, name: "Talk", diff --git a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts index 6ad94b7fbf..3b526f21ca 100644 --- a/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts +++ b/app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts @@ -47,6 +47,9 @@ export class LipSyncClip { this.blendshape_helper = new BlendShapeHelper(0, 0, 0); // we might need 3 of these one for each character ... this.lipsync = new LipSync(); + + + console.log("Lip!") } // lip sync will be generated through TTS @@ -95,6 +98,7 @@ async play(object: THREE.Object3D) { if (this.audio_data?.audioBuffer == null) { await this.download_audio(); } if (this.lipsync.face == null) { this.lipsync = new LipSync(await this._detect_face(object)); + console.log("Face", this.lipsync.face); this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); } } @@ -107,6 +111,7 @@ stop() { step() { if (this.lipsync == null) { return; } const positions = this.lipsync.update(); + console.log("Face", this.lipsync.face); this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"]; this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"]; this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"]; diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 23df4f9dcf..cf1d8390af 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -344,8 +344,6 @@ class Editor { ); } - console.log(this.cam_obj); - this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR, payload: { @@ -519,7 +517,7 @@ class Editor { async _serialize_timeline() { // note the database from the server is the source of truth for all the data. // Test code here - const object: any = await this.activeScene.load_glb( + const object: THREE.Object3D = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", ); @@ -649,9 +647,10 @@ class Editor { async _test_demo() { // note the database from the server is the source of truth for all the data. // Test code here - const object: any = await this.activeScene.load_glb( + const object: THREE.Object3D = await this.activeScene.load_glb( "./resources/models/fox/fox.glb", ); + object.uuid = "CH1"; // Load timeline creates the the clips from the datastructure and loads them in here. // load object into the engine for lip syncing diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index eea675f049..620f3f1511 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -173,12 +173,13 @@ export class TimeLine { public async addClip(data: any) { let object_uuid = data["data"]["object_uuid"]; - if (object_uuid.length < 6) { - object_uuid = this.scene.selected?.uuid; //data['data']['object_uuid']; - if (object_uuid == undefined) { - return; - } - } + // This is for testing for selected object instead of actual object given by timeline. + //if (object_uuid.length < 6) { + // object_uuid = this.scene.selected?.uuid; //data['data']['object_uuid']; + // if (object_uuid == undefined) { + // return; + // } + //} const media_id = data["data"]["media_id"]; const name = data["data"]["name"]; @@ -188,6 +189,35 @@ export class TimeLine { const offset = data["data"]["offset"]; const end_offset = data["data"]["length"] + offset; + switch (type) { + case "animation": + this.animation_engine.load_object(object_uuid, media_id, name); + break; + case "transform": + this.transform_engine.loadObject(object_uuid, data["data"]["length"]); + break; + case "audio": + if (group == "character") { + this.lipSync_engine.load_object(object_uuid, media_id); + // media id for this as well it can be downloaded + this.addPlayableClip( + new ClipUI( + version, + "lipsync", + group, + name, + media_id, + object_uuid, + offset, + end_offset)); + return; + } else { + console.log("Audio!") + this.audio_engine.loadClip(media_id); + } + break; + } + // media id for this as well it can be downloaded this.addPlayableClip( new ClipUI( @@ -201,15 +231,6 @@ export class TimeLine { end_offset, ), ); - - switch (type) { - case "animation": - this.animation_engine.load_object(object_uuid, media_id, name); - break; - case "transform": - this.transform_engine.loadObject(object_uuid, data["data"]["length"]); - break; - } } public async deleteKeyFrame(data: any) { @@ -228,6 +249,18 @@ export class TimeLine { public async updateClip(data: any) { // only length and offset changes here. console.log(data); + + let object_uuid = data["data"]["object_uuid"]; + const media_id = data["data"]["media_id"]; + const offset = data["data"]["offset"]; + const length = data["data"]["length"] + offset; + + for (const element of this.timeline_items) { + if (element.media_id == media_id && element.object_uuid == object_uuid) { + element.length = length; + element.offset = offset; + } + } } public async deleteClip(data: any) { console.log(data);