Added emotion engine needs the step functing working but cant work until i get json.

app/pages/PageEnigma/datastructures/clips/emotion_clip.ts

@@ -0,0 +1,81 @@
+import * as THREE from "three";
+
+export class EmotionClip  {
+  version: number;
+  media_id: string;
+  type: "emotion" = "emotion";
+  emotion_json: any;
+  faces: THREE.Mesh[];
+
+  constructor(version: number, media_id: string) {
+    this.version = version;
+    this.media_id = media_id;
+    this.type = "emotion";
+    this.faces = [];
+    this.download_json().then(data => {
+      this.emotion_json = data;
+    });
+  }
+
+  async get_media_url() {
+    //This is for prod when we have the proper info on the url.
+    let api_base_url = "https://api.fakeyou.com";
+    let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
+    let responce = await fetch(url);
+    let json = await JSON.parse(await responce.text());
+    let bucketPath = json["media_file"]["public_bucket_path"];
+    let media_base_url = "https://storage.googleapis.com/vocodes-public"
+    let media_url = `${media_base_url}${bucketPath}`
+    return media_url;
+  }
+
+  async download_json() {
+    let url = await this.get_media_url();
+    const response = await fetch(url);
+    return await response.json();
+  }
+
+  async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
+    this.faces = [];
+    return new Promise((resolve) => {
+      object.traverse((c: THREE.Object3D) => {
+        if (c instanceof THREE.Mesh) {
+          if (c.morphTargetInfluences && c.morphTargetDictionary) {
+            const blendShapeIndexE = c.morphTargetDictionary["E"];
+            if (blendShapeIndexE != null) {
+              this.faces.push(c);
+              resolve(c);
+            }
+          }
+        }
+      });
+    });
+  }
+
+  setBlends(shapes: { [key: string]: number }) {
+    this.faces.forEach((element: THREE.Mesh) => {
+      if (element.morphTargetInfluences && shapes) {
+        Object.keys(shapes).forEach(key => {
+          const index = element.morphTargetDictionary?.[key];
+          if (typeof index === 'number' && element.morphTargetInfluences !== undefined) {
+            element.morphTargetInfluences[index] = shapes[key];
+          }
+        });
+      }
+    });
+  }
+  
+
+  async step(frame: number, object: THREE.Object3D) {
+    await this._detect_face(object);
+    // setBlends()
+  }
+
+  toJSON(): any {
+    return {
+      version: this.version,
+      media_id: this.media_id,
+      type: this.type,
+    };
+  }
+}

app/pages/PageEnigma/js/audio_engine.ts

@@ -1,4 +1,3 @@
-import { faL } from "@fortawesome/pro-solid-svg-icons";
 import { AudioClip } from "../datastructures/clips/audio_clip";
 
 class AudioEngine {

app/pages/PageEnigma/js/editor.ts

@@ -13,8 +13,11 @@ import { SAOPass } from "three/addons/postprocessing/SAOPass.js";
 import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js";
 import { BokehPass } from "three/addons/postprocessing/BokehPass.js";
 import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
+
 import AudioEngine from "./audio_engine.js";
 import TransformEngine from "./transform_engine.js";
+import EmotionEngine from "./emotion_engine";
+
 import { TimeLine } from "./timeline.js";
 import { ClipUI } from "../datastructures/clips/clip_ui.js";
 import { LipSyncEngine } from "./lip_sync_engine.js";
@@ -77,6 +80,7 @@ class Editor {
   max_length: number;
   audio_engine: AudioEngine;
   transform_engine: TransformEngine;
+  emotion_engine: EmotionEngine;
   lipsync_engine: LipSyncEngine;
   animation_engine: AnimationEngine;
   timeline: TimeLine;
@@ -108,9 +112,6 @@ class Editor {
   recorder: MediaRecorder | undefined;
   // Default params.
 
-  // scene proxy for serialization
-  storyteller_proxy_scene: StoryTellerProxyScene;
-
   constructor() {
     console.log(
       "If you see this message twice! then it rendered twice, if you see it once it's all good.",
@@ -188,6 +189,7 @@ class Editor {
     this.canvasRenderCamReference;
 
     this.audio_engine = new AudioEngine();
+    this.emotion_engine = new EmotionEngine(this.version);
     this.transform_engine = new TransformEngine(this.version);
     this.lipsync_engine = new LipSyncEngine();
     this.animation_engine = new AnimationEngine(this.version);
@@ -198,6 +200,7 @@ class Editor {
       this.transform_engine,
       this.lipsync_engine,
       this.animation_engine,
+      this.emotion_engine,
       this.activeScene,
       this.camera,
       this.mouse,
@@ -217,11 +220,6 @@ class Editor {
       "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.";
     this.negative_prompt = "";
     this.art_style = ArtStyle.Anime2DFlat;
-
-    this.storyteller_proxy_scene = new StoryTellerProxyScene(
-      this.version,
-      this.activeScene.scene,
-    );
   }
 
   isEmpty(value: string) {
@@ -230,7 +228,7 @@ class Editor {
     );
   }
 
-  initialize(config: any, sceneToken) {
+  initialize(config: any, sceneToken: any) {
     //setup reactland Callbacks
     this.dispatchAppUiState = config.dispatchAppUiState;
 
@@ -252,6 +250,8 @@ class Editor {
     // Find the container element
     const container = document.getElementById("video-scene-container");
 
+    if(container == null) { return; }
+
     // Use the container's dimensions
     const width = container.offsetWidth;
     const height = container.offsetHeight;

app/pages/PageEnigma/js/emotion_engine.ts

@@ -0,0 +1,20 @@
+import { EmotionClip } from "../datastructures/clips/emotion_clip";
+
+class EmotionEngine {
+    clips: { [key: string]: EmotionClip } = {};
+    audio_sources: { [key: string]: AudioBufferSourceNode } = {};
+    version: number;
+
+    constructor(version: number) {
+        this.clips = {};
+        this.version = version;
+    }
+
+    // loads clips into the engine to cache
+    loadClip(object_uuid: string, audio_media_id: string) {
+        if(this.clips[audio_media_id+object_uuid] != null) { return; }
+        this.clips[audio_media_id+object_uuid] = new EmotionClip(this.version, audio_media_id);
+    }
+}
+
+export default EmotionEngine;

app/pages/PageEnigma/js/timeline.ts

@@ -16,6 +16,7 @@ import { fromEngineActions } from "../Queue/fromEngineActions";
 import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track";
 import { MediaItem } from "~/pages/PageEnigma/models";
 import Editor from "~/pages/PageEnigma/js/editor";
+import EmotionEngine from "./emotion_engine";
 
 // Every object uuid / entity has a track.
 export class TimelineDataState {
@@ -47,6 +48,10 @@ export class TimeLine {
   animation_engine: AnimationEngine;
   // lip sync engine
   lipSync_engine: LipSyncEngine;
+  // emotion engine
+  emotion_engine: EmotionEngine;
+
+  // characters
   characters: { [key: string]: ClipGroup };
 
   scene: Scene;
@@ -62,6 +67,7 @@ export class TimeLine {
     transform_engine: TransformEngine,
     lipsync_engine: LipSyncEngine,
     animation_engine: AnimationEngine,
+    emotion_engine: EmotionEngine,
     scene: Scene,
     camera: THREE.Camera,
     mouse: THREE.Vector2 | undefined
@@ -82,6 +88,7 @@ export class TimeLine {
     this.transform_engine = transform_engine;
     this.lipSync_engine = lipsync_engine;
     this.animation_engine = animation_engine;
+    this.emotion_engine = emotion_engine;
 
     this.scene = scene;
 
@@ -364,9 +371,9 @@ export class TimeLine {
       keyframe_offset,
     );
     this.transform_engine.clips[object_uuid].setTransform(
-      keyframe_uuid, 
-      keyframe_pos, 
-      keyframe_rot, 
+      keyframe_uuid,
+      keyframe_pos,
+      keyframe_rot,
       keyframe_scl,
     );
   }
@@ -588,6 +595,12 @@ export class TimeLine {
               this.scrubber_frame_position / fps, // Double FPS for best result.
             );
           }
+        } else if (element.type === ClipType.EMOTION) {
+          if (object) {
+            await this.emotion_engine.clips[
+              object.uuid + element.media_id
+            ].step(this.scrubber_frame_position-element.offset, object);
+          }
         }
         //this.timelineItems = this.timelineItems.filter(item => item !== element)
       }

app/pages/PageEnigma/models/track.ts

@@ -17,6 +17,7 @@ export enum ClipType {
   TRANSFORM = "transform",
   AUDIO = "audio",
   ANIMATION = "animation",
+  EMOTION = "emotion",
   FAKE = "fake",
 }