completed lipsync.

app/pages/PageEnigma/datastructures/clips/clip_offset.ts

@@ -1,16 +1,6 @@
-export interface ClipUI {
-  version: number
-  name: string
-  type: "transform" | "audio" | "animation" | "lipsync"
-  media_id: string
-  object_uuid: string
-  start_offset: number // in frames so 60 frames per second on the timeline.
-  ending_offset: number // ending offset
-}
-
 // Clip offsets represent the state of the clip on the timeline as well as what type of clip it is.
 // it is created from a media id.
-export class ClipUI implements ClipUI {
+export class ClipUI {
   version: number
   type: "transform" | "audio" | "animation" | "lipsync"
   name: string
@@ -21,8 +11,8 @@ export class ClipUI implements ClipUI {
 
   constructor(
     version: number,
-    name: string,
     type: "transform" | "audio" | "animation" | "lipsync",
+    name: string,
     media_id: string,
     object_uuid: string,
     start_offset: number,

app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts

@@ -1,99 +1,134 @@
 import { LipSync } from "../../js/lipsync";
 import * as THREE from 'three';
 interface AudioData {
-    audioContext: AudioContext;
-    audioBuffer: AudioBuffer;
+  audioContext: AudioContext;
+  audioBuffer: AudioBuffer;
+}
+
+class AudioData implements AudioData {
+  audioContext: AudioContext;
+  audioBuffer: AudioBuffer;
+  source: AudioBufferSourceNode | undefined
+
+  constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) {
+    this.audioContext = audioContext;
+    this.audioBuffer = audioBuffer;
   }
-  
-  class AudioData implements AudioData{
-    audioContext: AudioContext;
-    audioBuffer: AudioBuffer;
-    source: AudioBufferSourceNode | undefined
-  
-    constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) {
-      this.audioContext = audioContext;
-      this.audioBuffer = audioBuffer;
-    }
+}
+
+export interface LipSyncTrackClip {
+  version: number;
+  media_id: string;
+  type: "lipsync";
+  volume: number;
+}
+
+class BlendShapeHelper {
+  ah: number;
+  ee: number;
+  oh: number;
+  constructor(ah: number, ee: number, oh: number) {
+    this.ah = ah;
+    this.ee = ee;
+    this.oh = oh;
   }
-  
-  export interface LipSyncTrackClip {
-    version: number;
-    media_id: string;
-    type: "lipsync";
-    volume: number;
+}
+
+export class LipSyncTrackClip implements LipSyncTrackClip {
+  version: number;
+  media_id: string;
+  type: "lipsync" = "lipsync";
+  volume: number;
+  audio_data: AudioData | undefined;
+  lipsync: LipSync;
+  blendshape_helper: BlendShapeHelper;
+
+  constructor(version: number, media_id: string, volume: number) {
+    this.version = version;
+    this.media_id = media_id;
+    this.type = "lipsync";
+    this.volume = volume;
+    this.download_audio().then(data => {
+      this.audio_data = data;
+    });
+    this.blendshape_helper = new BlendShapeHelper(0, 0, 0);
+    // we might need 3 of these one for each character ...
+    this.lipsync = new LipSync();
   }
-  
-  export class LipSyncTrackClip implements LipSyncTrackClip {
-    version: number;
-    media_id: string;
-    type: "lipsync" = "lipsync";
-    volume: number;
-    audio_data: AudioData | undefined;
-    lipsync: LipSync;
-  
-    constructor(version: number, media_id: string, volume: number) {
-      this.version = version;
-      this.media_id = media_id;
-      this.type = "lipsync";
-      this.volume = volume;
-      this.download_audio().then(data => {
-        this.audio_data = data;
+
+  // lip sync will be generated through TTS 
+  async get_media_url() {
+    //This is for prod when we have the proper info on the url.
+    let api_base_url = "https://api.fakeyou.com";
+    let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
+    let responce = await fetch(url);
+    let json = await JSON.parse(await responce.text());
+    let bucketPath = json["media_file"]["public_bucket_path"];
+    let media_base_url = "https://storage.googleapis.com/vocodes-public"
+    let media_url = `${media_base_url}${bucketPath}`
+    return media_url;
+  }
+
+  async download_audio() {
+    let url = await this.get_media_url();
+    const audioContext = new AudioContext();
+    const response = await fetch(url);
+    const arrayBuffer = await response.arrayBuffer();
+    const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+    return new AudioData(audioContext, audioBuffer);
+  }
+
+  async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
+    return new Promise((resolve) => {
+      object.traverse((c: THREE.Object3D) => {
+        if (c instanceof THREE.Mesh) {
+          if (c.morphTargetInfluences && c.morphTargetDictionary) {
+            const blendShapeIndexE = c.morphTargetDictionary["E"];
+            const blendShapeIndexO = c.morphTargetDictionary["O"];
+            const blendShapeIndexA = c.morphTargetDictionary["aa"];
+            if (blendShapeIndexE != null) {
+              this.blendshape_helper = new BlendShapeHelper(blendShapeIndexA, blendShapeIndexE, blendShapeIndexO);
+              resolve(c);
+              return;
+            }
+          }
+        }
       });
-
-      // we might need 3 of these one for each character ...
-      this.lipsync = new LipSync()
-    }
-    
-    // lip sync will be generated through TTS 
-    async get_media_url() {
-      //This is for prod when we have the proper info on the url.
-      let api_base_url = "https://api.fakeyou.com";
-      let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
-      let responce = await fetch(url);
-      let json = await JSON.parse(await responce.text());
-      let bucketPath = json["media_file"]["public_bucket_path"];
-      let media_base_url = "https://storage.googleapis.com/vocodes-public"
-      let media_url = `${media_base_url}${bucketPath}`
-      return media_url;
-    }
-  
-    async download_audio() {
-      let url = await this.get_media_url();
-      const audioContext = new AudioContext();
-      const response = await fetch(url);
-      const arrayBuffer = await response.arrayBuffer();
-      const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
-      return new AudioData(audioContext, audioBuffer);
-    }
-
-    play(face: THREE.Object3D) {
-      if (this.audio_data?.audioBuffer == null) { this.download_audio(); }
-      this.lipsync = new LipSync()
-      this.lipsync.face = face;
-      this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
-    }
-
-    stop() {
-      if(this.lipsync == null) { return; }
-      this.lipsync.destroy();
-    }
-
-    step() {
-      if(this.lipsync == null) { return; }
-      this.lipsync.update();
-    }
-
-    reset() {
-      this.lipsync = new LipSync()
-    }
-  
-    toJSON(): string {
-      return JSON.stringify({
-        version: this.version,
-        media_id: this.media_id,
-        type: this.type,
-        volume: this.volume,
-      });
-    }
+    });
   }
-  
+
+
+  async play(object: THREE.Object3D) {
+  if (this.audio_data?.audioBuffer == null) { await this.download_audio(); }
+  if (this.lipsync.face == null) {
+    this.lipsync = new LipSync(await this._detect_face(object));
+    this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
+  }
+}
+
+stop() {
+  if (this.lipsync == null) { return; }
+  this.lipsync.destroy();
+}
+
+step() {
+  if (this.lipsync == null) { return; }
+  const positions = this.lipsync.update();
+  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"];
+  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"];
+  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"];
+}
+
+reset() {
+  this.lipsync = new LipSync();
+}
+
+toJSON(): string {
+  return JSON.stringify({
+    version: this.version,
+    media_id: this.media_id,
+    type: this.type,
+    volume: this.volume,
+  });
+}
+}

app/pages/PageEnigma/js/editor.js

@@ -155,8 +155,6 @@ class Editor {
 
         this.timeline.scene = this.activeScene;
         
-
-     
         this._test_demo()
     }
 
@@ -169,7 +167,7 @@ class Editor {
 
         let object = await this.activeScene.load_glb("./resources/models/fox/fox.glb")
         this.lipsync_engine.load_object(object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72");
-        this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", object.object_uuid, object.media_id, 0, 100));
+        this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 10, 200));
     }
 
     // Configure post processing.
@@ -278,11 +276,12 @@ class Editor {
 
         //console.log(this.transform_engine.clips[this.test_box_uuid]);
 
-        this.timeline.update(this.clock.getDelta());
+        this.timeline.update(this.clock.getDelta()).catch(data => {
+            console.log(data);
+        });
 
         this.render_scene();
         if (this.capturer != null) { this.capturer.capture(this.renderer.domElement); } // Record scene.
-
     }
 
     start_playback() {

app/pages/PageEnigma/js/lipsync.js

@@ -18,7 +18,7 @@ for (let m = 0; m < BoundingFrequencyFem.length; m++) {
 }
 
 export class LipSync {
-  constructor(face=null) {
+  constructor(face) {
 
     this.face = face;
 
@@ -90,19 +90,15 @@ export class LipSync {
   update() {
     if (this.meter) {
       const { volume } = this.meter;
+      let { ah, oh, ee } = this.update2();
       if (volume < 0.01) {
-        // get the shape keys from the face avatar
-
-        this.face.morphTargetInfluences[this.ee] = 0;
-        this.face.morphTargetInfluences[this.ah] = 0;
-        this.face.morphTargetInfluences[this.oh] = 0;
-      } else {
-        const { ah, oh, ee } = this.update2();
-        this.face.morphTargetInfluences[this.ee] = ee;
-        this.face.morphTargetInfluences[this.ah] = ah;
-        this.face.morphTargetInfluences[this.oh] = oh;
+        ah = 0;
+        oh = 0;
+        ee = 0;
       }
+      return { "ah": ah, "oh": oh, "ee": ee };
     }
+    return { "ah": 0, "oh": 0, "ee": 0 };
   }
 
   update2() {

app/pages/PageEnigma/js/timeline.ts

@@ -123,6 +123,9 @@ export class TimeLine {
                 this.audioEngine.loadClip(element.media_id);
             }
             else if (element.type == "animation") {
+            }
+            else if (element.type == "lipsync") {
+                
             }
             else {
                 this.stop()
@@ -140,10 +143,8 @@ export class TimeLine {
         }
 
         this.scrubberPosition += 1;
-
         //2. allow stopping.
         //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline.
-
         for (const element of this.timelineItems) {
             if (element.start_offset <= this.scrubberPosition && this.scrubberPosition <= element.ending_offset) {
                 // run async
@@ -161,15 +162,15 @@ export class TimeLine {
                     this.audioEngine.playClip(element.media_id)
                 }
                 else if (element.type == "lipsync") {
-                    let face_object = this.scene.get_object_by_uuid(element.object_uuid);
-                    if (face_object) { 
-                        this.lipSyncEngine.clips[element.object_uuid].play(face_object);
+                    if(this.scrubberPosition+1 >= element.ending_offset){
+                        this.lipSyncEngine.clips[element.object_uuid].stop();
+                    }
+                    else if (object) {
+                        await this.lipSyncEngine.clips[element.object_uuid].play(object);
                         this.lipSyncEngine.clips[element.object_uuid].step();
                     }
                 }
-
                 else if (element.type == "animation") {
-                    let object = this.scene.get_object_by_uuid(element.object_uuid);
                     if (object) { 
                         await this.animationEngine.clips[object.uuid].play(object); 
                         this.animationEngine.clips[object.uuid].step(deltatime);