Merge pull request #78 from storytold/audio-clips-for-timeline

Audio clips for timeline

app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx

@@ -126,7 +126,7 @@ export const TrackProvider = ({ children }: Props) => {
       },
       {
         version: 1,
-        media_id: uuid.v4(),
+        media_id: "m_403phjvjkbbaxxbz8y7r6qjay07mfd",
         type: ClipType.AUDIO,
         length: 25,
         name: "Talk",

app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts

@@ -47,6 +47,9 @@ export class LipSyncClip {
     this.blendshape_helper = new BlendShapeHelper(0, 0, 0);
     // we might need 3 of these one for each character ...
     this.lipsync = new LipSync();
+
+
+    console.log("Lip!")
   }
 
   // lip sync will be generated through TTS 
@@ -95,6 +98,7 @@ async play(object: THREE.Object3D) {
   if (this.audio_data?.audioBuffer == null) { await this.download_audio(); }
   if (this.lipsync.face == null) {
     this.lipsync = new LipSync(await this._detect_face(object));
+    console.log("Face", this.lipsync.face);
     this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
   }
 }
@@ -107,6 +111,7 @@ stop() {
 step() {
   if (this.lipsync == null) { return; }
   const positions = this.lipsync.update();
+  console.log("Face", this.lipsync.face);
   this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"];
   this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"];
   this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"];

app/pages/PageEnigma/js/editor.ts

@@ -344,8 +344,6 @@ class Editor {
       );
     }
 
-    console.log(this.cam_obj);
-
     this.dispatchAppUiState({
       type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR,
       payload: {
@@ -519,7 +517,7 @@ class Editor {
   async _serialize_timeline() {
     // note the database from the server is the source of truth for all the data.
     // Test code here
-    const object: any = await this.activeScene.load_glb(
+    const object: THREE.Object3D = await this.activeScene.load_glb(
       "./resources/models/fox/fox.glb",
     );
 
@@ -649,9 +647,10 @@ class Editor {
   async _test_demo() {
     // note the database from the server is the source of truth for all the data.
     // Test code here
-    const object: any = await this.activeScene.load_glb(
+    const object: THREE.Object3D = await this.activeScene.load_glb(
       "./resources/models/fox/fox.glb",
     );
+    object.uuid = "CH1";
 
     // Load timeline creates the the clips from the datastructure and loads them in here.
     // load object into the engine for lip syncing

app/pages/PageEnigma/js/timeline.ts

@@ -173,12 +173,13 @@ export class TimeLine {
     public async addClip(data: any) {
         let object_uuid = data["data"]["object_uuid"];
 
-        if (object_uuid.length < 6) {
-            object_uuid = this.scene.selected?.uuid; //data['data']['object_uuid'];
-            if (object_uuid == undefined) {
-                return;
-            }
-        }
+        // This is for testing for selected object instead of actual object given by timeline.
+        //if (object_uuid.length < 6) {
+        //    object_uuid = this.scene.selected?.uuid; //data['data']['object_uuid'];
+        //    if (object_uuid == undefined) {
+        //        return;
+        //    }
+        //}
 
         const media_id = data["data"]["media_id"];
         const name = data["data"]["name"];
@@ -188,6 +189,35 @@ export class TimeLine {
         const offset = data["data"]["offset"];
         const end_offset = data["data"]["length"] + offset;
 
+        switch (type) {
+            case "animation":
+                this.animation_engine.load_object(object_uuid, media_id, name);
+                break;
+            case "transform":
+                this.transform_engine.loadObject(object_uuid, data["data"]["length"]);
+                break;
+            case "audio":
+                if (group == "character") {
+                    this.lipSync_engine.load_object(object_uuid, media_id);
+                    // media id for this as well it can be downloaded
+                    this.addPlayableClip(
+                        new ClipUI(
+                            version,
+                            "lipsync",
+                            group,
+                            name,
+                            media_id,
+                            object_uuid,
+                            offset,
+                            end_offset));
+                    return;
+                } else {
+                    console.log("Audio!")
+                    this.audio_engine.loadClip(media_id);
+                }
+                break;
+        }
+
         // media id for this as well it can be downloaded
         this.addPlayableClip(
             new ClipUI(
@@ -201,15 +231,6 @@ export class TimeLine {
                 end_offset,
             ),
         );
-
-        switch (type) {
-            case "animation":
-                this.animation_engine.load_object(object_uuid, media_id, name);
-                break;
-            case "transform":
-                this.transform_engine.loadObject(object_uuid, data["data"]["length"]);
-                break;
-        }
     }
 
     public async deleteKeyFrame(data: any) {
@@ -228,6 +249,18 @@ export class TimeLine {
     public async updateClip(data: any) {
         // only length and offset changes here.
         console.log(data);
+
+        let object_uuid = data["data"]["object_uuid"];
+        const media_id = data["data"]["media_id"];
+        const offset = data["data"]["offset"];
+        const length = data["data"]["length"] + offset;
+
+        for (const element of this.timeline_items) {
+            if (element.media_id == media_id && element.object_uuid == object_uuid) {
+                element.length = length;
+                element.offset = offset;
+            }
+        }
     }
     public async deleteClip(data: any) {
         console.log(data);