From 37774d1e1c38de15aab2fc31f07e00e4555cc341 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Fri, 31 Jan 2025 15:50:41 -0800 Subject: [PATCH 1/2] Character Pose Mapper attempts --- .../Engines/Helpers/CharacterPoseHelper.ts | 288 +++++++++++++++++- .../Engines/Mappers/MixamoPoseMapper.ts | 48 +++ app/pages/PageEnigma/Editor/timeline.ts | 4 + .../comps/Timeline/CharacterFrameButtons.tsx | 20 +- 4 files changed, 352 insertions(+), 8 deletions(-) create mode 100644 app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts diff --git a/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts b/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts index 4e7a1a9952..841c4f6715 100644 --- a/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts +++ b/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts @@ -1,7 +1,12 @@ import { loadImage } from "~/Helpers/ImageHelpers"; import { EditorExpandedI } from "~/pages/PageEnigma/contexts/EngineContext"; import { FilesetResolver, HandLandmarker, HandLandmarkerResult, PoseLandmarker, PoseLandmarkerResult } from "@mediapipe/tasks-vision" +import { MixamoInterpolationBoneNames, MixamoPoseMap } from "../Mappers/MixamoPoseMapper"; +import { AxesHelper, Bone, Box3, Box3Helper, BufferGeometry, Euler, Matrix4, Object3D, Object3DEventMap, Quaternion, SkeletonHelper, Vector3 } from "three"; +// TODO: Currently the class uses the scene to use as a detachment parent +// Maybe this would cause problems in the future +// Look into an alternative way to detach bone from parent temporarily because reattaching? export class CharacterPoseHelper { editor: EditorExpandedI; @@ -26,7 +31,7 @@ export class CharacterPoseHelper { ); } - async extractPoseData(url: string): { hands: HandLandmarkerResult, pose: PoseLandmarkerResult } { + async extractPoseData(url: string): Promise<{ hands: HandLandmarkerResult; pose: PoseLandmarkerResult; }> { const frameImage = new Image(); frameImage.crossOrigin = "anonymous"; frameImage.src = url; @@ -68,14 +73,291 @@ export class CharacterPoseHelper { applyPoseDataToCharacter(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) { // Apply the pose data to the character - const character = this.editor.timeline.scene.get_object_by_uuid(characterId); + const scene = this.editor.timeline.scene; + const character = scene.get_object_by_uuid(characterId); if (!character) { console.error("Character not found with id: ", characterId); return; } // Apply the pose data to the character - console.debug("Character:", character) + // + // Get character dimensions with a bounding box (unreliable but works for now) + const characterBox = new Box3().setFromObject(character); + + // Add a box helper to the scene + const boxHelper = new Box3Helper(characterBox, 0xfff000); + scene.scene.add(boxHelper); + + // Get the original hip position + const hipPos = new Vector3(); + const hipBone = character.children[0].children.find((child) => child.name === "mixamorigHips")!; + hipBone.getWorldPosition(hipPos); + + // Test only the rotation without pose application for now + // TODO: Remove + this.updateBoneRotations(character); + return; + + // We'll do a triple pass, one that'll gather the bone data + // and another that'll update the main mapped bones and then finally interpolate the extra bones + // + // First pass: Build a list of bones to update + const mappedBones: Record> = {}; + const interpolationBones: Record> = {}; + character.traverse((child) => { + if (child.type !== "Bone") { + return; + } + + if (MixamoPoseMap[child.name] !== undefined) { + console.debug("Mapped bone found:", child.name); + mappedBones[child.name] = child; + } else { + // console.debug("Interpolation bone found:", child.name); + interpolationBones[child.name] = child; + } + }); + + // Second pass: Update the mapped bones + const scale = 1.1; + Object.values(mappedBones).forEach((bone) => { + const mapIndex = MixamoPoseMap[bone.name]; + const poseCoordinates = poseData.pose.worldLandmarks[0][mapIndex]; + const parent = bone.parent!; + + // Detach child from parent + scene.scene.attach(bone); + + bone.position.set(poseCoordinates.x * scale, -poseCoordinates.y * scale, -poseCoordinates.z * scale); + bone.position.add(hipPos); // Pose data world coordinates are centered at center of hips (left and right) + + // Reattach child to its parent + parent.attach(bone); + }); + + // Third pass: Interpolate the extra bones + if (interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh]) { + // Left thigh is lipo'd between left upper leg and knee + const leftThigh = interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh]; + const leftUpperLeg = mappedBones["mixamorigLeftUpLeg"]; + const leftKnee = mappedBones["mixamorigLeftLeg"]; + const parent = leftThigh.parent!; + + // Detach child from parent + scene.scene.attach(leftThigh); + + // Calculate the lipo position + const leftUpperLegPos = new Vector3(); + const leftKneePos = new Vector3(); + leftUpperLeg.getWorldPosition(leftUpperLegPos); + console.debug("Left upper leg pos: ", leftUpperLegPos); + leftKnee.getWorldPosition(leftKneePos); + console.debug("Left knee pos: ", leftKneePos); + const leftThighPos = new Vector3(); + leftThighPos.lerpVectors(leftUpperLegPos, leftKneePos, 0.6); + console.debug("Left thigh pos: ", leftThighPos); + leftThigh.position.set(leftThighPos.x, leftThighPos.y, leftThighPos.z); + + // Reattach child to its parent + parent.attach(leftThigh); + } + + if (interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm]) { + // Right upper arm is lipo'd between shoulder and elbow + const rightUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm]; + const rightShoulder = mappedBones["mixamorigRightArm"]; + const rightElbow = mappedBones["mixamorigRightForeArm"]; + const parent = rightUpperArm.parent!; + + // Detach child from parent + scene.scene.attach(rightUpperArm); + + // Calculate the lipo position + const rightShoulderPos = new Vector3(); + const rightElbowPos = new Vector3(); + rightShoulder.getWorldPosition(rightShoulderPos); + console.debug("Right shoulder pos: ", rightShoulderPos); + rightElbow.getWorldPosition(rightElbowPos); + console.debug("Right elbow pos: ", rightElbowPos); + const rightUpperArmPos = new Vector3(); + rightUpperArmPos.lerpVectors(rightShoulderPos, rightElbowPos, 0.5); + console.debug("Right upper arm pos: ", rightUpperArmPos); + rightUpperArm.position.set(rightUpperArmPos.x, rightUpperArmPos.y, rightUpperArmPos.z); + + // Reattach child to its parent + parent.attach(rightUpperArm); + } + + if (interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm]) { + // Left upper arm is lipo'd between shoulder and elbow + const leftUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm]; + console.log("Left def arm position:", leftUpperArm.position); + const leftShoulder = mappedBones["mixamorigLeftArm"]; + const leftElbow = mappedBones["mixamorigLeftForeArm"]; + const parent = leftUpperArm.parent!; + + // Detach child from parent + scene.scene.attach(leftUpperArm); + + // Calculate the lipo position + const leftShoulderPos = new Vector3(); + const leftElbowPos = new Vector3(); + leftShoulder.getWorldPosition(leftShoulderPos); + console.debug("Left shoulder pos: ", leftShoulderPos); + leftElbow.getWorldPosition(leftElbowPos); + console.debug("Left elbow pos: ", leftElbowPos); + const leftUpperArmPos = new Vector3(); + leftUpperArmPos.lerpVectors(leftShoulderPos, leftElbowPos, 0.5); + console.debug("Left upper arm pos: ", leftUpperArmPos); + leftUpperArm.position.set(leftUpperArmPos.x, leftUpperArmPos.y, leftUpperArmPos.z); + + // Reattach child to its parent + parent.attach(leftUpperArm); + console.log("Left def arm position:", leftUpperArm.position); + } + + // this.updateBoneRotations(character); + } + + updateBoneRotations(character: Object3D) { + const skeletonHelper = new SkeletonHelper(character); + const rootBone = skeletonHelper.bones[0]; + const queue = [rootBone]; + const scene = this.editor.timeline.scene; + + const fromBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftArm")!; + const toBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftForeArm")!; + const fromHelper = new AxesHelper(10); + const toHelper = new AxesHelper(10); + fromHelper.material.transparent = true; + fromHelper.material.depthTest = false; + fromBone.add(fromHelper); + toBone.add(toHelper); + + console.debug("Before rotations: ", fromBone, toBone); + this.updateBoneRotationEuler(fromBone, toBone); + console.debug("After rotations: ", fromBone, toBone); + } + + updateBoneRotationEuler(bone: Bone, childBone: Bone) { + // Get world positions + const boneWorldPos = new Vector3(); + const childWorldPos = new Vector3(); + bone.getWorldPosition(boneWorldPos); + childBone.getWorldPosition(childWorldPos); + + // Compute the new direction the bone should point to + const targetDirection = new Vector3().subVectors(childWorldPos, boneWorldPos).normalize(); + console.debug("Target direction: ", targetDirection); + + // Convert world direction to local space (relative to bone) + const parentMatrix = new Matrix4(); + if (bone.parent) { + bone.parent.updateMatrixWorld(true); + parentMatrix.copy(bone.matrixWorld).invert(); // Convert to local space + } + targetDirection.applyMatrix4(parentMatrix); + console.debug("Target direction local: ", targetDirection); + + // Compute Euler angles from the target direction + const eulerRotation = new Euler(); + eulerRotation.setFromVector3(targetDirection); + + // Apply rotation + bone.rotation.copy(eulerRotation); + bone.updateMatrixWorld(true); + } + + getAlignmentMap(character: Object3D, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) { + const alignmentMap: Record = {}; + + character.traverse((child) => { + if (child.type !== "Bone") { + return; + } + + if (!MixamoPoseMap[child.name]) { + return; + } + + const mapIndex = MixamoPoseMap[child.name]; + const targetCoordinates = poseData.pose.worldLandmarks[0][mapIndex]; + const targetWorldCoordinates = new Vector3( + targetCoordinates.x, -targetCoordinates.y, -targetCoordinates.z + ); + + alignmentMap[child.name] = targetWorldCoordinates; + }); + + return alignmentMap; + } + + alignCharacterToPose(bones: Bone[], targetPositions: Record) { + const tempVec1 = new Vector3(); + const tempVec2 = new Vector3(); + const quat = new Quaternion(); + + bones.forEach(bone => { + if (!targetPositions[bone.name]) return; // Skip if no target is given + + const targetPos = targetPositions[bone.name]; + + // Get current world position + const currentPos = bone.getWorldPosition(new Vector3()); + + // Compute position offset (not always needed for Mixamo rigs) + const positionOffset = tempVec1.subVectors(targetPos, currentPos); + bone.position.add(positionOffset); // Only apply if necessary + + if (bone.parent && bone.parent.type === "Bone") { + // Compute world-space bone direction before and after transformation + const childPos = bone.getWorldPosition(new Vector3()); + const parentPos = bone.parent.getWorldPosition(new Vector3()); + const parentQuat = bone.parent.getWorldQuaternion(new Quaternion()); + + // Compute direction vectors + tempVec1.subVectors(childPos, parentPos).normalize(); // Current direction + tempVec2.subVectors(targetPos, parentPos).normalize(); // Target direction + + // Compute rotation needed to align current direction to target direction + quat.setFromUnitVectors(tempVec1, tempVec2); + + // Convert world-space rotation to local-space + const inverseParentQuat = parentQuat.clone().invert(); + quat.premultiply(inverseParentQuat); + + bone.quaternion.premultiply(quat); + } + }); + } + + testRun(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) { + const scene = this.editor.timeline.scene; + const character = scene.get_object_by_uuid(characterId); + if (!character) { + console.error("Character not found with id: ", characterId); + return; + } + const alignmentMap = this.getAlignmentMap(character, poseData); + const sortedBones = this.getSortedBoneArray(character); + this.alignCharacterToPose(sortedBones, alignmentMap); + } + + getSortedBoneArray(character: Object3D) { + function getBoneHierarchyDepth(bone: Bone, root: Bone): number { + let depth = 0; + while (bone !== root && bone.parent) { + bone = bone.parent as Bone; + depth++; + } + return depth; + } + + const skeletonHelper = new SkeletonHelper(character); + const rootBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigHips")!; + skeletonHelper.bones.sort((a, b) => getBoneHierarchyDepth(a, rootBone) - getBoneHierarchyDepth(b, rootBone)); + return skeletonHelper.bones; } } diff --git a/app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts b/app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts new file mode 100644 index 0000000000..e1749f6cd3 --- /dev/null +++ b/app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts @@ -0,0 +1,48 @@ +// A mapping between the model bones names and the Mediapipe output index +export const MixamoPoseMap: Record = { + "mixamorigLeftArm": 11, + "mixamorigRightArm": 12, + "mixamorigLeftForeArm": 13, + "mixamorigRightForeArm": 14, + "mixamorigLeftHand": 15, + "mixamorigRightHand": 16, + "mixamorigLeftUpLeg": 23, + "mixamorigRightUpLeg": 24, + "mixamorigLeftLeg": 25, + "mixamorigRightLeg": 26, + "mixamorigLeftFoot": 27, + "mixamorigRightFoot": 28, + "mixamorigLeftHandThumb2": 21, + "mixamorigRightHandThumb2": 22, +} + +export const MixamoInterpolationBoneNames = { + Spine: "mixamorigSpine", + Spine1: "mixamorigSpine1", + Spine2: "mixamorigSpine2", + Neck: "mixamorigNeck", + LeftShoulderBlade: "mixamorigLeftShoulder", + DefLeftUpperArm: "DEF-upper_armL001", + DefLeftForeArm: "DEF-forearmL001", + RightShoulderBlade: "mixamorigRightShoulder", + DefRightUpperArm: "DEF-upper_armR001", + DefRightForeArm: "DEF-forearmR001", + DefLeftThigh: "DEF-thighL001", + DefRightThigh: "DEF-thighR001", +} + +export const PoseMap = { + NOSE: 0, + LEFT_SHOULDER: 11, + RIGHT_SHOULDER: 12, + LEFT_ELBOW: 13, + RIGHT_ELBOW: 14, + LEFT_WRIST: 15, + RIGHT_WRIST: 16, + LEFT_HIP: 23, + RIGHT_HIP: 24, + LEFT_KNEE: 25, + RIGHT_KNEE: 26, + LEFT_ANKLE: 27, + RIGHT_ANKLE: 28, +} diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index 356e123097..4036da9036 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -273,6 +273,10 @@ export class TimeLine { console.debug("Added character object") console.debug("object:", obj) + // Add a skeleton helper to the character + const skeletonHelper = new THREE.SkeletonHelper(obj); + this.scene.scene.add(skeletonHelper); + const object_uuid = obj.uuid; this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character. diff --git a/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx b/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx index 7c25976bfc..59bb24a0dd 100644 --- a/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx +++ b/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx @@ -1,7 +1,7 @@ import { faImage, faTrashCan } from "@fortawesome/pro-solid-svg-icons"; import { ButtonIconStack } from "~/components/reusable/ButtonIconStack"; import { frameTrackButtonWidthPx } from "../../signals"; -import { useCallback, useContext, useRef, useState } from "react"; +import { useCallback, useContext, useEffect, useRef, useState } from "react"; import { EngineContext } from "../../contexts/EngineContext"; import { CHARACTER_FRAME_FILE_TYPE } from "~/enums"; import { UploadModalMedia } from "~/components/reusable/UploadModalMedia"; @@ -60,6 +60,16 @@ export default function CharacterFrameButton( setIsUploadModalOpen(false); }, [setIsUploadModalOpen]); + // eslint-disable-next-line react-hooks/exhaustive-deps + const handleTest = () => { + async function test() { + const poseHelper = new CharacterPoseHelper(editorEngine!); + const poseData = await poseHelper.extractPoseData("https://cdn-2.fakeyou.com/media/y/6/q/s/v/y6qsvxh66qw9bq8a48619t5pg9sdeyzx/image_y6qsvxh66qw9bq8a48619t5pg9sdeyzx.png") + // poseHelper.applyPoseDataToCharacter(characterId, poseData); + poseHelper.testRun(characterId, poseData); + } + test(); + }; const handleFrameSet = useCallback((token?: string) => { if (!token) { @@ -72,9 +82,9 @@ export default function CharacterFrameButton( get_media_url(token) .then((url) => { console.debug("Frame url: ", url) + // Handle the pose data - const poseHelper = new CharacterPoseHelper(editorEngine!); - poseHelper.extractPoseData(url); + handleTest(); setMediaFile(url); }) .catch((error) => { @@ -84,7 +94,7 @@ export default function CharacterFrameButton( unlockButton(); }) - }, [unlockButton, setMediaFile]); + }, [unlockButton, setMediaFile, handleTest]); const handleDeleteFrame = useCallback(() => { setMediaFile(undefined); @@ -96,7 +106,7 @@ export default function CharacterFrameButton( return ( <>
- +
Date: Fri, 31 Jan 2025 15:51:03 -0800 Subject: [PATCH 2/2] Save manager userdata field support --- app/pages/PageEnigma/Editor/editor.ts | 2 +- app/pages/PageEnigma/Editor/save_manager.ts | 4 +- .../proxy/storyteller_proxy_3d_object.ts | 11 ++-- .../proxy/storyteller_proxy_scene.ts | 6 +-- .../proxy/storyteller_proxy_timeline.ts | 50 +++++++------------ 5 files changed, 33 insertions(+), 40 deletions(-) diff --git a/app/pages/PageEnigma/Editor/editor.ts b/app/pages/PageEnigma/Editor/editor.ts index 8f4c136c2b..ea2eceb0ce 100644 --- a/app/pages/PageEnigma/Editor/editor.ts +++ b/app/pages/PageEnigma/Editor/editor.ts @@ -213,7 +213,7 @@ class Editor { // life cycle fix // Version and name. - this.version = 1.1; + this.version = 1.2; // Clock, scene and camera essentials. // global names this.camera_name = "::CAM::"; diff --git a/app/pages/PageEnigma/Editor/save_manager.ts b/app/pages/PageEnigma/Editor/save_manager.ts index 7ec0b5715f..819feeb609 100644 --- a/app/pages/PageEnigma/Editor/save_manager.ts +++ b/app/pages/PageEnigma/Editor/save_manager.ts @@ -181,6 +181,7 @@ export class SaveManager { this.editor.generating_preview = false; // FIX THIS LATER WITH VICCCCCCCCCCCCCCCTORRRRRRRR + console.debug("Save Scene Result: ", result); return result; // if this is an empty string it is an error. need to migrate to api manager. } @@ -260,8 +261,9 @@ export class SaveManager { this.editor.lipsync_engine, this.editor.emotion_engine, ); + console.debug("Loading Timeline", scene_json["timeline"]); await proxyTimeline.loadFromJson(scene_json["timeline"]); - + this.editor.timeline.checkEditorCanPlay(); hideEditorLoader(); diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts index 285fa61847..fb99c0ecb1 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts @@ -28,7 +28,8 @@ export interface ObjectJSON { specular: number; locked: boolean; visible: boolean | undefined; - media_file_type: MediaFileType; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + user_data: Record; } export class StoryTellerProxy3DObject { @@ -48,7 +49,8 @@ export class StoryTellerProxy3DObject { specular: number; locked: boolean; visible: boolean; - media_file_type: MediaFileType; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + userData: Record; constructor(version: number, media_file_token: string) { this.version = version; @@ -69,7 +71,8 @@ export class StoryTellerProxy3DObject { this.specular = 0.5; this.locked = false; this.visible = true; - this.media_file_type = MediaFileType.None; + this.userData = {}; + this.userData.media_file_type = MediaFileType.None; } getColorAsHexString(object: THREE.Object3D): string { @@ -125,7 +128,7 @@ export class StoryTellerProxy3DObject { specular: this.specular, locked: this.locked, visible: this.visible, - media_file_type: this.media_file_type, + user_data: this.userData, }; return json; } diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index c4ba45b629..b4f8d351b7 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -48,9 +48,7 @@ export class StoryTellerProxyScene { proxyObject3D.specular = child.userData["specular"]; proxyObject3D.locked = child.userData["locked"]; proxyObject3D.visible = child.visible; - if (child.userData["media_file_type"] !== undefined) { - proxyObject3D.media_file_type = child.userData["media_file_type"]; - } + proxyObject3D.userData = child.userData; const json_data = await proxyObject3D.toJSON(); return json_data; } @@ -98,6 +96,7 @@ export class StoryTellerProxyScene { for (const pchild of this.scene.scene.children) { if (this.version >= 1.0) { if (pchild.userData["media_id"] != undefined) { + console.debug("Object JSON:", pchild.toJSON()) results.push(await this.getChildren(pchild)); } } else { @@ -108,6 +107,7 @@ export class StoryTellerProxyScene { } else { console.log("Scene doesn't exist needs to be assigned"); } + console.debug("Scene saved.", results); return results; } diff --git a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts index 5af843f04d..c99a3a6d99 100644 --- a/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts +++ b/app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts @@ -10,6 +10,8 @@ import EmotionEngine from "../Editor/Engines/emotion_engine"; import LipSyncEngine from "../Editor/Engines/lip_sync_engine"; import { TimeLine } from "../Editor/timeline"; import TransformEngine from "../Editor/Engines/transform_engine"; +import Ijson from "~/interfaces/Ijson"; +import { ClipGroup, ClipType } from "~/enums"; export class StoryTellerProxyTimeline { timeline: TimeLine; @@ -36,7 +38,7 @@ export class StoryTellerProxyTimeline { this.emotion_engine = emotion_engine; } - private async getItemsToJson(items: unknown[]): Promise { + private async getItemsToJson(items: Ijson[]): Promise { const timeline_items_data: unknown[] = []; items.forEach((element) => { timeline_items_data.push(element.toJSON()); }); @@ -44,11 +46,11 @@ export class StoryTellerProxyTimeline { } private async getItemsDict(items: { - [key: string]: any; - }): Promise<{ [key: string]: any }> { - const timeline_items_data: { [key: string]: any } = {}; + [key: string]: Ijson; + }): Promise<{ [key: string]: unknown }> { + const timeline_items_data: { [key: string]: unknown } = {}; for (const key in items) { - if (items.hasOwnProperty(key)) { + if (Object.prototype.hasOwnProperty.call(items, key)) { const element = items[key]; timeline_items_data[key] = element.toJSON(); } @@ -60,7 +62,7 @@ export class StoryTellerProxyTimeline { const timeline_json = { timeline: await this.getItemsToJson(this.timeline.timeline_items), transform: await this.getItemsDict(this.transform_engine.clips), - animation: await this.getItemsDict(this.animation_engine.clips), + animation: this.animation_engine.toJSON(), audio: await this.getItemsDict(this.audio_engine.clips), lipsync: await this.getItemsDict(this.lipsync_engine.clips), emotion: await this.getItemsDict(this.emotion_engine.clips), @@ -131,27 +133,6 @@ export class StoryTellerProxyTimeline { return timeline_items_data; } - private async loadAnimationClips(items: { - [key: string]: any; - }): Promise<{ [key: string]: any }> { - const timeline_items_data: { [key: string]: any } = {}; - for (const key in items) { - if (items.hasOwnProperty(key)) { - const element = items[key]; - timeline_items_data[key] = new AnimationClip( - element.version, - element.media_id, - "remote", - element.object_uuid, - element.speed, - element.length, - element.clip_name, - ); - } - } - return timeline_items_data; - } - private async loadLipsyncClips(items: { [key: string]: any; }): Promise<{ [key: string]: any }> { @@ -196,14 +177,21 @@ export class StoryTellerProxyTimeline { this.lipsync_engine.clips = await this.loadLipsyncClips( timeline["lipsync"], ); - this.animation_engine.clips = await this.loadAnimationClips( - timeline["animation"], - ); this.emotion_engine.clips = await this.loadEmotionClips( timeline["emotion"], ); - console.log(this.timeline.timeline_items); + // Add characters and their animations + const animationClips = this.timeline.timeline_items.filter((clip) => { clip.group === ClipGroup.CHARACTER && clip.type === ClipType.ANIMATION }) + animationClips.forEach((clip) => { + const characterObject = this.timeline.scene.get_object_by_uuid(clip.object_uuid); + + if (!characterObject) { + return + } + + this.animation_engine.addCharacterAnimationMedia(characterObject, clip.media_id, clip); + }); this.timeline.updateUI(); }