Merge branch 'tanish/frames' into deployable

app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts

@@ -1,7 +1,12 @@
 import { loadImage } from "~/Helpers/ImageHelpers";
 import { EditorExpandedI } from "~/pages/PageEnigma/contexts/EngineContext";
 import { FilesetResolver, HandLandmarker, HandLandmarkerResult, PoseLandmarker, PoseLandmarkerResult } from "@mediapipe/tasks-vision"
+import { MixamoInterpolationBoneNames, MixamoPoseMap } from "../Mappers/MixamoPoseMapper";
+import { AxesHelper, Bone, Box3, Box3Helper, BufferGeometry, Euler, Matrix4, Object3D, Object3DEventMap, Quaternion, SkeletonHelper, Vector3 } from "three";
 
+// TODO: Currently the class uses the scene to use as a detachment parent
+// Maybe this would cause problems in the future
+// Look into an alternative way to detach bone from parent temporarily because reattaching?
 export class CharacterPoseHelper {
 
   editor: EditorExpandedI;
@@ -26,7 +31,7 @@ export class CharacterPoseHelper {
     );
   }
 
-  async extractPoseData(url: string): { hands: HandLandmarkerResult, pose: PoseLandmarkerResult } {
+  async extractPoseData(url: string): Promise<{ hands: HandLandmarkerResult; pose: PoseLandmarkerResult; }> {
     const frameImage = new Image();
     frameImage.crossOrigin = "anonymous";
     frameImage.src = url;
@@ -68,14 +73,291 @@ export class CharacterPoseHelper {
 
   applyPoseDataToCharacter(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) {
     // Apply the pose data to the character
-    const character = this.editor.timeline.scene.get_object_by_uuid(characterId);
+    const scene = this.editor.timeline.scene;
+    const character = scene.get_object_by_uuid(characterId);
     if (!character) {
       console.error("Character not found with id: ", characterId);
       return;
     }
 
     // Apply the pose data to the character
-    console.debug("Character:", character)
+    //
+    // Get character dimensions with a bounding box (unreliable but works for now)
+    const characterBox = new Box3().setFromObject(character);
+
+    // Add a box helper to the scene
+    const boxHelper = new Box3Helper(characterBox, 0xfff000);
+    scene.scene.add(boxHelper);
+
+    // Get the original hip position
+    const hipPos = new Vector3();
+    const hipBone = character.children[0].children.find((child) => child.name === "mixamorigHips")!;
+    hipBone.getWorldPosition(hipPos);
+
+    // Test only the rotation without pose application for now
+    // TODO: Remove
+    this.updateBoneRotations(character);
+    return;
+
+    // We'll do a triple pass, one that'll gather the bone data
+    // and another that'll update the main mapped bones and then finally interpolate the extra bones
+    //
+    // First pass: Build a list of bones to update
+    const mappedBones: Record<string, Object3D<Object3DEventMap>> = {};
+    const interpolationBones: Record<string, Object3D<Object3DEventMap>> = {};
+    character.traverse((child) => {
+      if (child.type !== "Bone") {
+        return;
+      }
+
+      if (MixamoPoseMap[child.name] !== undefined) {
+        console.debug("Mapped bone found:", child.name);
+        mappedBones[child.name] = child;
+      } else {
+        // console.debug("Interpolation bone found:", child.name);
+        interpolationBones[child.name] = child;
+      }
+    });
+
+    // Second pass: Update the mapped bones
+    const scale = 1.1;
+    Object.values(mappedBones).forEach((bone) => {
+      const mapIndex = MixamoPoseMap[bone.name];
+      const poseCoordinates = poseData.pose.worldLandmarks[0][mapIndex];
+      const parent = bone.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(bone);
+
+      bone.position.set(poseCoordinates.x * scale, -poseCoordinates.y * scale, -poseCoordinates.z * scale);
+      bone.position.add(hipPos); // Pose data world coordinates are centered at center of hips (left and right)
+
+      // Reattach child to its parent
+      parent.attach(bone);
+    });
+
+    // Third pass: Interpolate the extra bones
+    if (interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh]) {
+      // Left thigh is lipo'd between left upper leg and knee
+      const leftThigh = interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh];
+      const leftUpperLeg = mappedBones["mixamorigLeftUpLeg"];
+      const leftKnee = mappedBones["mixamorigLeftLeg"];
+      const parent = leftThigh.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(leftThigh);
+
+      // Calculate the lipo position
+      const leftUpperLegPos = new Vector3();
+      const leftKneePos = new Vector3();
+      leftUpperLeg.getWorldPosition(leftUpperLegPos);
+      console.debug("Left upper leg pos: ", leftUpperLegPos);
+      leftKnee.getWorldPosition(leftKneePos);
+      console.debug("Left knee pos: ", leftKneePos);
+      const leftThighPos = new Vector3();
+      leftThighPos.lerpVectors(leftUpperLegPos, leftKneePos, 0.6);
+      console.debug("Left thigh pos: ", leftThighPos);
+      leftThigh.position.set(leftThighPos.x, leftThighPos.y, leftThighPos.z);
+
+      // Reattach child to its parent
+      parent.attach(leftThigh);
+    }
+
+    if (interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm]) {
+      // Right upper arm is lipo'd between shoulder and elbow
+      const rightUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm];
+      const rightShoulder = mappedBones["mixamorigRightArm"];
+      const rightElbow = mappedBones["mixamorigRightForeArm"];
+      const parent = rightUpperArm.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(rightUpperArm);
+
+      // Calculate the lipo position
+      const rightShoulderPos = new Vector3();
+      const rightElbowPos = new Vector3();
+      rightShoulder.getWorldPosition(rightShoulderPos);
+      console.debug("Right shoulder pos: ", rightShoulderPos);
+      rightElbow.getWorldPosition(rightElbowPos);
+      console.debug("Right elbow pos: ", rightElbowPos);
+      const rightUpperArmPos = new Vector3();
+      rightUpperArmPos.lerpVectors(rightShoulderPos, rightElbowPos, 0.5);
+      console.debug("Right upper arm pos: ", rightUpperArmPos);
+      rightUpperArm.position.set(rightUpperArmPos.x, rightUpperArmPos.y, rightUpperArmPos.z);
+
+      // Reattach child to its parent
+      parent.attach(rightUpperArm);
+    }
+
+    if (interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm]) {
+      // Left upper arm is lipo'd between shoulder and elbow
+      const leftUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm];
+      console.log("Left def arm position:", leftUpperArm.position);
+      const leftShoulder = mappedBones["mixamorigLeftArm"];
+      const leftElbow = mappedBones["mixamorigLeftForeArm"];
+      const parent = leftUpperArm.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(leftUpperArm);
+
+      // Calculate the lipo position
+      const leftShoulderPos = new Vector3();
+      const leftElbowPos = new Vector3();
+      leftShoulder.getWorldPosition(leftShoulderPos);
+      console.debug("Left shoulder pos: ", leftShoulderPos);
+      leftElbow.getWorldPosition(leftElbowPos);
+      console.debug("Left elbow pos: ", leftElbowPos);
+      const leftUpperArmPos = new Vector3();
+      leftUpperArmPos.lerpVectors(leftShoulderPos, leftElbowPos, 0.5);
+      console.debug("Left upper arm pos: ", leftUpperArmPos);
+      leftUpperArm.position.set(leftUpperArmPos.x, leftUpperArmPos.y, leftUpperArmPos.z);
+
+      // Reattach child to its parent
+      parent.attach(leftUpperArm);
+      console.log("Left def arm position:", leftUpperArm.position);
+    }
+
+    // this.updateBoneRotations(character);
+  }
+
+  updateBoneRotations(character: Object3D<Object3DEventMap>) {
+    const skeletonHelper = new SkeletonHelper(character);
+    const rootBone = skeletonHelper.bones[0];
+    const queue = [rootBone];
+    const scene = this.editor.timeline.scene;
+
+    const fromBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftArm")!;
+    const toBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftForeArm")!;
+    const fromHelper = new AxesHelper(10);
+    const toHelper = new AxesHelper(10);
+    fromHelper.material.transparent = true;
+    fromHelper.material.depthTest = false;
+    fromBone.add(fromHelper);
+    toBone.add(toHelper);
+
+    console.debug("Before rotations: ", fromBone, toBone);
+    this.updateBoneRotationEuler(fromBone, toBone);
+    console.debug("After rotations: ", fromBone, toBone);
+  }
+
+  updateBoneRotationEuler(bone: Bone, childBone: Bone) {
+    // Get world positions
+    const boneWorldPos = new Vector3();
+    const childWorldPos = new Vector3();
+    bone.getWorldPosition(boneWorldPos);
+    childBone.getWorldPosition(childWorldPos);
+
+    // Compute the new direction the bone should point to
+    const targetDirection = new Vector3().subVectors(childWorldPos, boneWorldPos).normalize();
+    console.debug("Target direction: ", targetDirection);
+
+    // Convert world direction to local space (relative to bone)
+    const parentMatrix = new Matrix4();
+    if (bone.parent) {
+      bone.parent.updateMatrixWorld(true);
+      parentMatrix.copy(bone.matrixWorld).invert(); // Convert to local space
+    }
+    targetDirection.applyMatrix4(parentMatrix);
+    console.debug("Target direction local: ", targetDirection);
+
+    // Compute Euler angles from the target direction
+    const eulerRotation = new Euler();
+    eulerRotation.setFromVector3(targetDirection);
+
+    // Apply rotation
+    bone.rotation.copy(eulerRotation);
+    bone.updateMatrixWorld(true);
+  }
+
+  getAlignmentMap(character: Object3D<Object3DEventMap>, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) {
+    const alignmentMap: Record<string, Vector3> = {};
+
+    character.traverse((child) => {
+      if (child.type !== "Bone") {
+        return;
+      }
+
+      if (!MixamoPoseMap[child.name]) {
+        return;
+      }
+
+      const mapIndex = MixamoPoseMap[child.name];
+      const targetCoordinates = poseData.pose.worldLandmarks[0][mapIndex];
+      const targetWorldCoordinates = new Vector3(
+        targetCoordinates.x, -targetCoordinates.y, -targetCoordinates.z
+      );
+
+      alignmentMap[child.name] = targetWorldCoordinates;
+    });
+
+    return alignmentMap;
+  }
+
+  alignCharacterToPose(bones: Bone[], targetPositions: Record<string, Vector3>) {
+    const tempVec1 = new Vector3();
+    const tempVec2 = new Vector3();
+    const quat = new Quaternion();
+
+    bones.forEach(bone => {
+      if (!targetPositions[bone.name]) return;  // Skip if no target is given
+
+      const targetPos = targetPositions[bone.name];
+
+      // Get current world position
+      const currentPos = bone.getWorldPosition(new Vector3());
+
+      // Compute position offset (not always needed for Mixamo rigs)
+      const positionOffset = tempVec1.subVectors(targetPos, currentPos);
+      bone.position.add(positionOffset); // Only apply if necessary
+
+      if (bone.parent && bone.parent.type === "Bone") {
+        // Compute world-space bone direction before and after transformation
+        const childPos = bone.getWorldPosition(new Vector3());
+        const parentPos = bone.parent.getWorldPosition(new Vector3());
+        const parentQuat = bone.parent.getWorldQuaternion(new Quaternion());
+
+        // Compute direction vectors
+        tempVec1.subVectors(childPos, parentPos).normalize(); // Current direction
+        tempVec2.subVectors(targetPos, parentPos).normalize(); // Target direction
+
+        // Compute rotation needed to align current direction to target direction
+        quat.setFromUnitVectors(tempVec1, tempVec2);
+
+        // Convert world-space rotation to local-space
+        const inverseParentQuat = parentQuat.clone().invert();
+        quat.premultiply(inverseParentQuat);
+
+        bone.quaternion.premultiply(quat);
+      }
+    });
+  }
+
+  testRun(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) {
+    const scene = this.editor.timeline.scene;
+    const character = scene.get_object_by_uuid(characterId);
+    if (!character) {
+      console.error("Character not found with id: ", characterId);
+      return;
+    }
+    const alignmentMap = this.getAlignmentMap(character, poseData);
+    const sortedBones = this.getSortedBoneArray(character);
+    this.alignCharacterToPose(sortedBones, alignmentMap);
+  }
+
+  getSortedBoneArray(character: Object3D<Object3DEventMap>) {
+    function getBoneHierarchyDepth(bone: Bone, root: Bone): number {
+      let depth = 0;
+      while (bone !== root && bone.parent) {
+        bone = bone.parent as Bone;
+        depth++;
+      }
+      return depth;
+    }
+
+    const skeletonHelper = new SkeletonHelper(character);
+    const rootBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigHips")!;
+    skeletonHelper.bones.sort((a, b) => getBoneHierarchyDepth(a, rootBone) - getBoneHierarchyDepth(b, rootBone));
+    return skeletonHelper.bones;
   }
 
 }

app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts

@@ -0,0 +1,48 @@
+// A mapping between the model bones names and the Mediapipe output index
+export const MixamoPoseMap: Record<string, number> = {
+  "mixamorigLeftArm": 11,
+  "mixamorigRightArm": 12,
+  "mixamorigLeftForeArm": 13,
+  "mixamorigRightForeArm": 14,
+  "mixamorigLeftHand": 15,
+  "mixamorigRightHand": 16,
+  "mixamorigLeftUpLeg": 23,
+  "mixamorigRightUpLeg": 24,
+  "mixamorigLeftLeg": 25,
+  "mixamorigRightLeg": 26,
+  "mixamorigLeftFoot": 27,
+  "mixamorigRightFoot": 28,
+  "mixamorigLeftHandThumb2": 21,
+  "mixamorigRightHandThumb2": 22,
+}
+
+export const MixamoInterpolationBoneNames = {
+  Spine: "mixamorigSpine",
+  Spine1: "mixamorigSpine1",
+  Spine2: "mixamorigSpine2",
+  Neck: "mixamorigNeck",
+  LeftShoulderBlade: "mixamorigLeftShoulder",
+  DefLeftUpperArm: "DEF-upper_armL001",
+  DefLeftForeArm: "DEF-forearmL001",
+  RightShoulderBlade: "mixamorigRightShoulder",
+  DefRightUpperArm: "DEF-upper_armR001",
+  DefRightForeArm: "DEF-forearmR001",
+  DefLeftThigh: "DEF-thighL001",
+  DefRightThigh: "DEF-thighR001",
+}
+
+export const PoseMap = {
+  NOSE: 0,
+  LEFT_SHOULDER: 11,
+  RIGHT_SHOULDER: 12,
+  LEFT_ELBOW: 13,
+  RIGHT_ELBOW: 14,
+  LEFT_WRIST: 15,
+  RIGHT_WRIST: 16,
+  LEFT_HIP: 23,
+  RIGHT_HIP: 24,
+  LEFT_KNEE: 25,
+  RIGHT_KNEE: 26,
+  LEFT_ANKLE: 27,
+  RIGHT_ANKLE: 28,
+}

app/pages/PageEnigma/Editor/editor.ts

@@ -213,7 +213,7 @@ class Editor {
     // life cycle fix
 
     // Version and name.
-    this.version = 1.1;
+    this.version = 1.2;
     // Clock, scene and camera essentials.
     // global names
     this.camera_name = "::CAM::";

app/pages/PageEnigma/Editor/save_manager.ts

@@ -181,6 +181,7 @@ export class SaveManager {
 
     this.editor.generating_preview = false; // FIX THIS LATER WITH VICCCCCCCCCCCCCCCTORRRRRRRR
 
+    console.debug("Save Scene Result: ", result);
     return result; // if this is an empty string it is an error. need to migrate to api manager.
   }
 
@@ -260,8 +261,9 @@ export class SaveManager {
       this.editor.lipsync_engine,
       this.editor.emotion_engine,
     );
+    console.debug("Loading Timeline", scene_json["timeline"]);
     await proxyTimeline.loadFromJson(scene_json["timeline"]);
-    
+
     this.editor.timeline.checkEditorCanPlay();
 
     hideEditorLoader();

app/pages/PageEnigma/Editor/timeline.ts

@@ -287,6 +287,10 @@ export class TimeLine {
         testDeformBody(lastRig);
       }
 
+      // Add a skeleton helper to the character
+      const skeletonHelper = new THREE.SkeletonHelper(obj);
+      this.scene.scene.add(skeletonHelper);
+
       const object_uuid = obj.uuid;
 
       this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.

app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx

@@ -1,7 +1,7 @@
 import { faImage, faTrashCan } from "@fortawesome/pro-solid-svg-icons";
 import { ButtonIconStack } from "~/components/reusable/ButtonIconStack";
 import { frameTrackButtonWidthPx } from "../../signals";
-import { useCallback, useContext, useRef, useState } from "react";
+import { useCallback, useContext, useEffect, useRef, useState } from "react";
 import { EngineContext } from "../../contexts/EngineContext";
 import { CHARACTER_FRAME_FILE_TYPE } from "~/enums";
 import { UploadModalMedia } from "~/components/reusable/UploadModalMedia";
@@ -64,6 +64,16 @@ export default function CharacterFrameButton(
     setIsUploadModalOpen(false);
   }, [setIsUploadModalOpen]);
 
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  const handleTest = () => {
+    async function test() {
+      const poseHelper = new CharacterPoseHelper(editorEngine!);
+      const poseData = await poseHelper.extractPoseData("https://cdn-2.fakeyou.com/media/y/6/q/s/v/y6qsvxh66qw9bq8a48619t5pg9sdeyzx/image_y6qsvxh66qw9bq8a48619t5pg9sdeyzx.png")
+      // poseHelper.applyPoseDataToCharacter(characterId, poseData);
+      poseHelper.testRun(characterId, poseData);
+    }
+    test();
+  };
 
   const handleFrameSet = useCallback((token?: string) => {
     if (!token) {
@@ -90,6 +100,7 @@ export default function CharacterFrameButton(
           poseHelper.extractPoseData(url);
         }
 
+        handleTest();
         setMediaFile(url);
 
         // TODO(brandon,2024-01-27): Please forgive me for this ugly hack. It's just 
@@ -109,7 +120,7 @@ export default function CharacterFrameButton(
         unlockButton();
       })
 
-  }, [unlockButton, setMediaFile]);
+  }, [unlockButton, setMediaFile, handleTest]);
 
   const handleDeleteFrame = useCallback(() => {
     setMediaFile(undefined);
@@ -121,7 +132,7 @@ export default function CharacterFrameButton(
     return (
       <>
         <div className={className} style={{ minWidth: frameTrackButtonWidthPx, width: frameTrackButtonWidthPx }}>
-          <ButtonIconStack icon={faImage} additionalStyle="bg-character-frame" text={CharacterFrameStrings[target]} onClick={handleFrameClick} />
+          <ButtonIconStack icon={faImage} additionalStyle="bg-character-frame" text={CharacterFrameStrings[target]} onClick={handleTest} />
         </div>
         <UploadImageMediaModal
           isOpen={isUploadModalOpen}

app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts

@@ -28,7 +28,8 @@ export interface ObjectJSON {
   specular: number;
   locked: boolean;
   visible: boolean | undefined;
-  media_file_type: MediaFileType;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  user_data: Record<string, any>;
 }
 
 export class StoryTellerProxy3DObject {
@@ -48,7 +49,8 @@ export class StoryTellerProxy3DObject {
   specular: number;
   locked: boolean;
   visible: boolean;
-  media_file_type: MediaFileType;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  userData: Record<string, any>;
 
   constructor(version: number, media_file_token: string) {
     this.version = version;
@@ -69,7 +71,8 @@ export class StoryTellerProxy3DObject {
     this.specular = 0.5;
     this.locked = false;
     this.visible = true;
-    this.media_file_type = MediaFileType.None;
+    this.userData = {};
+    this.userData.media_file_type = MediaFileType.None;
   }
 
   getColorAsHexString(object: THREE.Object3D): string {
@@ -125,7 +128,7 @@ export class StoryTellerProxy3DObject {
       specular: this.specular,
       locked: this.locked,
       visible: this.visible,
-      media_file_type: this.media_file_type,
+      user_data: this.userData,
     };
     return json;
   }

app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts

@@ -48,9 +48,7 @@ export class StoryTellerProxyScene {
     proxyObject3D.specular = child.userData["specular"];
     proxyObject3D.locked = child.userData["locked"];
     proxyObject3D.visible = child.visible;
-    if (child.userData["media_file_type"] !== undefined) {
-      proxyObject3D.media_file_type = child.userData["media_file_type"];
-    }
+    proxyObject3D.userData = child.userData;
     const json_data = await proxyObject3D.toJSON();
     return json_data;
   }
@@ -98,6 +96,7 @@ export class StoryTellerProxyScene {
       for (const pchild of this.scene.scene.children) {
         if (this.version >= 1.0) {
           if (pchild.userData["media_id"] != undefined) {
+            console.debug("Object JSON:", pchild.toJSON())
             results.push(await this.getChildren(pchild));
           }
         } else {
@@ -108,6 +107,7 @@ export class StoryTellerProxyScene {
     } else {
       console.log("Scene doesn't exist needs to be assigned");
     }
+    console.debug("Scene saved.", results);
     return results;
   }
 

app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts

@@ -10,6 +10,8 @@ import EmotionEngine from "../Editor/Engines/emotion_engine";
 import LipSyncEngine from "../Editor/Engines/lip_sync_engine";
 import { TimeLine } from "../Editor/timeline";
 import TransformEngine from "../Editor/Engines/transform_engine";
+import Ijson from "~/interfaces/Ijson";
+import { ClipGroup, ClipType } from "~/enums";
 
 export class StoryTellerProxyTimeline {
   timeline: TimeLine;
@@ -36,7 +38,7 @@ export class StoryTellerProxyTimeline {
     this.emotion_engine = emotion_engine;
   }
 
-  private async getItemsToJson(items: unknown[]): Promise<unknown[]> {
+  private async getItemsToJson(items: Ijson[]): Promise<unknown[]> {
     const timeline_items_data: unknown[] = [];
     items.forEach((element) => { timeline_items_data.push(element.toJSON()); });
 
@@ -44,11 +46,11 @@ export class StoryTellerProxyTimeline {
   }
 
   private async getItemsDict(items: {
-    [key: string]: any;
-  }): Promise<{ [key: string]: any }> {
-    const timeline_items_data: { [key: string]: any } = {};
+    [key: string]: Ijson;
+  }): Promise<{ [key: string]: unknown }> {
+    const timeline_items_data: { [key: string]: unknown } = {};
     for (const key in items) {
-      if (items.hasOwnProperty(key)) {
+      if (Object.prototype.hasOwnProperty.call(items, key)) {
         const element = items[key];
         timeline_items_data[key] = element.toJSON();
       }
@@ -60,7 +62,7 @@ export class StoryTellerProxyTimeline {
     const timeline_json = {
       timeline: await this.getItemsToJson(this.timeline.timeline_items),
       transform: await this.getItemsDict(this.transform_engine.clips),
-      animation: await this.getItemsDict(this.animation_engine.clips),
+      animation: this.animation_engine.toJSON(),
       audio: await this.getItemsDict(this.audio_engine.clips),
       lipsync: await this.getItemsDict(this.lipsync_engine.clips),
       emotion: await this.getItemsDict(this.emotion_engine.clips),
@@ -131,27 +133,6 @@ export class StoryTellerProxyTimeline {
     return timeline_items_data;
   }
 
-  private async loadAnimationClips(items: {
-    [key: string]: any;
-  }): Promise<{ [key: string]: any }> {
-    const timeline_items_data: { [key: string]: any } = {};
-    for (const key in items) {
-      if (items.hasOwnProperty(key)) {
-        const element = items[key];
-        timeline_items_data[key] = new AnimationClip(
-          element.version,
-          element.media_id,
-          "remote",
-          element.object_uuid,
-          element.speed,
-          element.length,
-          element.clip_name,
-        );
-      }
-    }
-    return timeline_items_data;
-  }
-
   private async loadLipsyncClips(items: {
     [key: string]: any;
   }): Promise<{ [key: string]: any }> {
@@ -196,14 +177,21 @@ export class StoryTellerProxyTimeline {
     this.lipsync_engine.clips = await this.loadLipsyncClips(
       timeline["lipsync"],
     );
-    this.animation_engine.clips = await this.loadAnimationClips(
-      timeline["animation"],
-    );
     this.emotion_engine.clips = await this.loadEmotionClips(
       timeline["emotion"],
     );
 
-    console.log(this.timeline.timeline_items);
+    // Add characters and their animations
+    const animationClips = this.timeline.timeline_items.filter((clip) => { clip.group === ClipGroup.CHARACTER && clip.type === ClipType.ANIMATION })
+    animationClips.forEach((clip) => {
+      const characterObject = this.timeline.scene.get_object_by_uuid(clip.object_uuid);
+
+      if (!characterObject) {
+        return
+      }
+
+      this.animation_engine.addCharacterAnimationMedia(characterObject, clip.media_id, clip);
+    });
 
     this.timeline.updateUI();
   }