Merge pull request #83 from storytold/feature/pose-mode

Feature/pose mode

app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx

@@ -103,7 +103,7 @@ export const TrackProvider = ({ children }: Props) => {
       },
       {
         version: 1,
-        media_id: uuid.v4(),
+        media_id: "m_w5t517krrf63f3tj2288vsffmj87zw",
         type: ClipType.ANIMATION,
         length: 25,
         name: "Walk",

app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts

@@ -35,6 +35,7 @@ export class LipSyncClip {
   audio_data: AudioData | undefined;
   lipsync: LipSync;
   blendshape_helper: BlendShapeHelper;
+  faces: THREE.Mesh[]
 
   constructor(version: number, media_id: string, volume: number) {
     this.version = version;
@@ -47,9 +48,7 @@ export class LipSyncClip {
     this.blendshape_helper = new BlendShapeHelper(0, 0, 0);
     // we might need 3 of these one for each character ...
     this.lipsync = new LipSync();
-
-
-    console.log("Lip!")
+    this.faces = [];
   }
 
   // lip sync will be generated through TTS 
@@ -84,8 +83,8 @@ export class LipSyncClip {
             const blendShapeIndexA = c.morphTargetDictionary["aa"];
             if (blendShapeIndexE != null) {
               this.blendshape_helper = new BlendShapeHelper(blendShapeIndexA, blendShapeIndexE, blendShapeIndexO);
+              this.faces.push(c);
               resolve(c);
-              return;
             }
           }
         }
@@ -94,44 +93,48 @@ export class LipSyncClip {
   }
 
 
-async play(object: THREE.Object3D) {
-  if (this.audio_data?.audioBuffer == null) { await this.download_audio(); }
-  if (this.lipsync.face == null) {
-    this.lipsync = new LipSync(await this._detect_face(object));
-    console.log("Face", this.lipsync.face);
-    this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
+  async play(object: THREE.Object3D) {
+    if (this.audio_data?.audioBuffer == null) { await this.download_audio(); }
+    if (this.lipsync.face == null) {
+      this.lipsync = new LipSync(await this._detect_face(object));
+      this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
+    }
+  }
+
+  stop() {
+    if (this.lipsync == null) { return; }
+    this.lipsync.destroy();
+  }
+
+  setBlends(ah: number, ee: number, oh: number) {
+    this.faces.forEach((element: THREE.Mesh) => {
+      if (element.morphTargetInfluences) {
+        element.morphTargetInfluences[this.blendshape_helper.ee] = ee;
+        element.morphTargetInfluences[this.blendshape_helper.ah] = ah;
+        element.morphTargetInfluences[this.blendshape_helper.oh] = oh;
+      }
+    });
+  }
+
+  step() {
+    if (this.lipsync == null) { return; }
+    const positions = this.lipsync.update();
+    this.setBlends(positions["ee"], positions["ah"], positions["oh"]);
+  }
+
+  reset() {
+    if (this.lipsync.face != undefined) {
+      this.setBlends(0,0,0);
+    }
+    this.lipsync = new LipSync();
+  }
+
+  toJSON(): string {
+    return JSON.stringify({
+      version: this.version,
+      media_id: this.media_id,
+      type: this.type,
+      volume: this.volume,
+    });
   }
 }
-
-stop() {
-  if (this.lipsync == null) { return; }
-  this.lipsync.destroy();
-}
-
-step() {
-  if (this.lipsync == null) { return; }
-  const positions = this.lipsync.update();
-  console.log("Face", this.lipsync.face);
-  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"];
-  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"];
-  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"];
-}
-
-reset() {
-  if (this.lipsync.face != undefined) {
-    this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = 0;
-    this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = 0
-    this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = 0;
-  }
-  this.lipsync = new LipSync();
-}
-
-toJSON(): string {
-  return JSON.stringify({
-    version: this.version,
-    media_id: this.media_id,
-    type: this.type,
-    volume: this.volume,
-  });
-}
-}

app/pages/PageEnigma/js/editor.ts

@@ -652,7 +652,7 @@ class Editor {
     // note the database from the server is the source of truth for all the data.
     // Test code here
     const object: THREE.Object3D = await this.activeScene.load_glb(
-      "./resources/models/fox/fox.glb",
+      "m_9f3d3z94kk6m25zywyz6an3p43fjtw",
     );
     object.uuid = "CH1";
 
@@ -851,6 +851,9 @@ class Editor {
         } else if(this.timeline.is_playing == false) {
           this.cam_obj.position.copy(this.camera.position);
           this.cam_obj.rotation.copy(this.camera.rotation);
+        } else {
+          this.camera.position.copy(this.cam_obj.position);
+          this.camera.rotation.copy(this.cam_obj.rotation);
         }
       }
     }
@@ -1019,9 +1022,9 @@ class Editor {
     URL.revokeObjectURL(url);
     document.body.removeChild(downloadLink);
 
-    const data = await this.api_manager.uploadMedia(blob, "tmp.wav");
-    console.log(data);
+    const data = await this.api_manager.uploadMedia(blob, "render.mp4");
     // Create a link to download the file
+
   }
 
   async generateFrame() {

app/pages/PageEnigma/js/scene.ts

@@ -77,7 +77,7 @@ class Scene {
     }
 
     _create_camera_obj() {
-        this.load_glb("/resources/models/camera/camera.glb", false).then((cam_obj) => {
+        this.load_glb("m_cxh4asqhapdz10j880755dg4yevshb", false).then((cam_obj) => {
             cam_obj.userData["name"] = "::CAM::";
             cam_obj.name = "::CAM::";
             cam_obj.position.set(0, 0.6, 1.5);
@@ -97,7 +97,45 @@ class Scene {
         }
     }
 
-    async load_glb(filepath: string, auto_add: boolean = true): Promise<THREE.Object3D> { //: Promise<THREE.Object3D> {
+    async getMediaURL(media_id: string) {
+        //This is for prod when we have the proper info on the url.
+        let api_base_url = "https://api.fakeyou.com";
+        let url = `${api_base_url}/v1/media_files/file/${media_id}`
+        let responce = await fetch(url);
+        let json = await JSON.parse(await responce.text());
+        let bucketPath = json["media_file"]["public_bucket_path"];
+        let media_base_url = "https://storage.googleapis.com/vocodes-public"
+        let media_url = `${media_base_url}${bucketPath}`
+        return media_url;
+    }
+
+    async load_glb(media_id: string, auto_add: boolean = true): Promise<THREE.Object3D> { //: Promise<THREE.Object3D> {
+        return new Promise(async (resolve) => {
+            let glbLoader = new GLTFLoader();
+            glbLoader.load(await this.getMediaURL(media_id), (glb) => {
+                glb.scene.children.forEach(child => {
+                    child.traverse((c: THREE.Object3D) => {
+                        if (c instanceof THREE.Mesh) {
+                            c.material.metalness = 0.0;
+                            c.material.specular = 0.5;
+                            c.castShadow = true;
+                            c.receiveShadow = true;
+                            c.material.transparent = false;
+                        }
+                    });
+                    if (child.type == "Group") {
+                        if (auto_add) { this.scene.add(child.children[0]); }
+                        resolve(child.children[0]);
+                        return;
+                    }
+                    if (auto_add) { this.scene.add(child); }
+                    resolve(child);
+                });
+            });
+        });
+    }
+
+    async load_glb_absolute(filepath: string, auto_add: boolean = true): Promise<THREE.Object3D> { //: Promise<THREE.Object3D> {
         return new Promise((resolve) => {
             let glbLoader = new GLTFLoader();
             glbLoader.load(filepath, (glb) => {

app/pages/PageEnigma/js/timeline.ts

@@ -392,7 +392,7 @@ export class TimeLine {
                     if (object) {
                         await this.animation_engine.clips[object.uuid].play(object);
                         this.animation_engine.clips[object.uuid].step(
-                            this.scrubber_frame_position / 60,
+                            this.scrubber_frame_position / 120, // Double FPS for best result.
                         );
                     }
                 } else {