Character Pose Mapper attempts

app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts

@@ -1,7 +1,12 @@
 import { loadImage } from "~/Helpers/ImageHelpers";
 import { EditorExpandedI } from "~/pages/PageEnigma/contexts/EngineContext";
 import { FilesetResolver, HandLandmarker, HandLandmarkerResult, PoseLandmarker, PoseLandmarkerResult } from "@mediapipe/tasks-vision"
+import { MixamoInterpolationBoneNames, MixamoPoseMap } from "../Mappers/MixamoPoseMapper";
+import { AxesHelper, Bone, Box3, Box3Helper, BufferGeometry, Euler, Matrix4, Object3D, Object3DEventMap, Quaternion, SkeletonHelper, Vector3 } from "three";
 
+// TODO: Currently the class uses the scene to use as a detachment parent
+// Maybe this would cause problems in the future
+// Look into an alternative way to detach bone from parent temporarily because reattaching?
 export class CharacterPoseHelper {
 
   editor: EditorExpandedI;
@@ -26,7 +31,7 @@ export class CharacterPoseHelper {
     );
   }
 
-  async extractPoseData(url: string): { hands: HandLandmarkerResult, pose: PoseLandmarkerResult } {
+  async extractPoseData(url: string): Promise<{ hands: HandLandmarkerResult; pose: PoseLandmarkerResult; }> {
     const frameImage = new Image();
     frameImage.crossOrigin = "anonymous";
     frameImage.src = url;
@@ -68,14 +73,291 @@ export class CharacterPoseHelper {
 
   applyPoseDataToCharacter(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) {
     // Apply the pose data to the character
-    const character = this.editor.timeline.scene.get_object_by_uuid(characterId);
+    const scene = this.editor.timeline.scene;
+    const character = scene.get_object_by_uuid(characterId);
     if (!character) {
       console.error("Character not found with id: ", characterId);
       return;
     }
 
     // Apply the pose data to the character
-    console.debug("Character:", character)
+    //
+    // Get character dimensions with a bounding box (unreliable but works for now)
+    const characterBox = new Box3().setFromObject(character);
+
+    // Add a box helper to the scene
+    const boxHelper = new Box3Helper(characterBox, 0xfff000);
+    scene.scene.add(boxHelper);
+
+    // Get the original hip position
+    const hipPos = new Vector3();
+    const hipBone = character.children[0].children.find((child) => child.name === "mixamorigHips")!;
+    hipBone.getWorldPosition(hipPos);
+
+    // Test only the rotation without pose application for now
+    // TODO: Remove
+    this.updateBoneRotations(character);
+    return;
+
+    // We'll do a triple pass, one that'll gather the bone data
+    // and another that'll update the main mapped bones and then finally interpolate the extra bones
+    //
+    // First pass: Build a list of bones to update
+    const mappedBones: Record<string, Object3D<Object3DEventMap>> = {};
+    const interpolationBones: Record<string, Object3D<Object3DEventMap>> = {};
+    character.traverse((child) => {
+      if (child.type !== "Bone") {
+        return;
+      }
+
+      if (MixamoPoseMap[child.name] !== undefined) {
+        console.debug("Mapped bone found:", child.name);
+        mappedBones[child.name] = child;
+      } else {
+        // console.debug("Interpolation bone found:", child.name);
+        interpolationBones[child.name] = child;
+      }
+    });
+
+    // Second pass: Update the mapped bones
+    const scale = 1.1;
+    Object.values(mappedBones).forEach((bone) => {
+      const mapIndex = MixamoPoseMap[bone.name];
+      const poseCoordinates = poseData.pose.worldLandmarks[0][mapIndex];
+      const parent = bone.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(bone);
+
+      bone.position.set(poseCoordinates.x * scale, -poseCoordinates.y * scale, -poseCoordinates.z * scale);
+      bone.position.add(hipPos); // Pose data world coordinates are centered at center of hips (left and right)
+
+      // Reattach child to its parent
+      parent.attach(bone);
+    });
+
+    // Third pass: Interpolate the extra bones
+    if (interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh]) {
+      // Left thigh is lipo'd between left upper leg and knee
+      const leftThigh = interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh];
+      const leftUpperLeg = mappedBones["mixamorigLeftUpLeg"];
+      const leftKnee = mappedBones["mixamorigLeftLeg"];
+      const parent = leftThigh.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(leftThigh);
+
+      // Calculate the lipo position
+      const leftUpperLegPos = new Vector3();
+      const leftKneePos = new Vector3();
+      leftUpperLeg.getWorldPosition(leftUpperLegPos);
+      console.debug("Left upper leg pos: ", leftUpperLegPos);
+      leftKnee.getWorldPosition(leftKneePos);
+      console.debug("Left knee pos: ", leftKneePos);
+      const leftThighPos = new Vector3();
+      leftThighPos.lerpVectors(leftUpperLegPos, leftKneePos, 0.6);
+      console.debug("Left thigh pos: ", leftThighPos);
+      leftThigh.position.set(leftThighPos.x, leftThighPos.y, leftThighPos.z);
+
+      // Reattach child to its parent
+      parent.attach(leftThigh);
+    }
+
+    if (interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm]) {
+      // Right upper arm is lipo'd between shoulder and elbow
+      const rightUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm];
+      const rightShoulder = mappedBones["mixamorigRightArm"];
+      const rightElbow = mappedBones["mixamorigRightForeArm"];
+      const parent = rightUpperArm.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(rightUpperArm);
+
+      // Calculate the lipo position
+      const rightShoulderPos = new Vector3();
+      const rightElbowPos = new Vector3();
+      rightShoulder.getWorldPosition(rightShoulderPos);
+      console.debug("Right shoulder pos: ", rightShoulderPos);
+      rightElbow.getWorldPosition(rightElbowPos);
+      console.debug("Right elbow pos: ", rightElbowPos);
+      const rightUpperArmPos = new Vector3();
+      rightUpperArmPos.lerpVectors(rightShoulderPos, rightElbowPos, 0.5);
+      console.debug("Right upper arm pos: ", rightUpperArmPos);
+      rightUpperArm.position.set(rightUpperArmPos.x, rightUpperArmPos.y, rightUpperArmPos.z);
+
+      // Reattach child to its parent
+      parent.attach(rightUpperArm);
+    }
+
+    if (interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm]) {
+      // Left upper arm is lipo'd between shoulder and elbow
+      const leftUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm];
+      console.log("Left def arm position:", leftUpperArm.position);
+      const leftShoulder = mappedBones["mixamorigLeftArm"];
+      const leftElbow = mappedBones["mixamorigLeftForeArm"];
+      const parent = leftUpperArm.parent!;
+
+      // Detach child from parent
+      scene.scene.attach(leftUpperArm);
+
+      // Calculate the lipo position
+      const leftShoulderPos = new Vector3();
+      const leftElbowPos = new Vector3();
+      leftShoulder.getWorldPosition(leftShoulderPos);
+      console.debug("Left shoulder pos: ", leftShoulderPos);
+      leftElbow.getWorldPosition(leftElbowPos);
+      console.debug("Left elbow pos: ", leftElbowPos);
+      const leftUpperArmPos = new Vector3();
+      leftUpperArmPos.lerpVectors(leftShoulderPos, leftElbowPos, 0.5);
+      console.debug("Left upper arm pos: ", leftUpperArmPos);
+      leftUpperArm.position.set(leftUpperArmPos.x, leftUpperArmPos.y, leftUpperArmPos.z);
+
+      // Reattach child to its parent
+      parent.attach(leftUpperArm);
+      console.log("Left def arm position:", leftUpperArm.position);
+    }
+
+    // this.updateBoneRotations(character);
+  }
+
+  updateBoneRotations(character: Object3D<Object3DEventMap>) {
+    const skeletonHelper = new SkeletonHelper(character);
+    const rootBone = skeletonHelper.bones[0];
+    const queue = [rootBone];
+    const scene = this.editor.timeline.scene;
+
+    const fromBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftArm")!;
+    const toBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftForeArm")!;
+    const fromHelper = new AxesHelper(10);
+    const toHelper = new AxesHelper(10);
+    fromHelper.material.transparent = true;
+    fromHelper.material.depthTest = false;
+    fromBone.add(fromHelper);
+    toBone.add(toHelper);
+
+    console.debug("Before rotations: ", fromBone, toBone);
+    this.updateBoneRotationEuler(fromBone, toBone);
+    console.debug("After rotations: ", fromBone, toBone);
+  }
+
+  updateBoneRotationEuler(bone: Bone, childBone: Bone) {
+    // Get world positions
+    const boneWorldPos = new Vector3();
+    const childWorldPos = new Vector3();
+    bone.getWorldPosition(boneWorldPos);
+    childBone.getWorldPosition(childWorldPos);
+
+    // Compute the new direction the bone should point to
+    const targetDirection = new Vector3().subVectors(childWorldPos, boneWorldPos).normalize();
+    console.debug("Target direction: ", targetDirection);
+
+    // Convert world direction to local space (relative to bone)
+    const parentMatrix = new Matrix4();
+    if (bone.parent) {
+      bone.parent.updateMatrixWorld(true);
+      parentMatrix.copy(bone.matrixWorld).invert(); // Convert to local space
+    }
+    targetDirection.applyMatrix4(parentMatrix);
+    console.debug("Target direction local: ", targetDirection);
+
+    // Compute Euler angles from the target direction
+    const eulerRotation = new Euler();
+    eulerRotation.setFromVector3(targetDirection);
+
+    // Apply rotation
+    bone.rotation.copy(eulerRotation);
+    bone.updateMatrixWorld(true);
+  }
+
+  getAlignmentMap(character: Object3D<Object3DEventMap>, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) {
+    const alignmentMap: Record<string, Vector3> = {};
+
+    character.traverse((child) => {
+      if (child.type !== "Bone") {
+        return;
+      }
+
+      if (!MixamoPoseMap[child.name]) {
+        return;
+      }
+
+      const mapIndex = MixamoPoseMap[child.name];
+      const targetCoordinates = poseData.pose.worldLandmarks[0][mapIndex];
+      const targetWorldCoordinates = new Vector3(
+        targetCoordinates.x, -targetCoordinates.y, -targetCoordinates.z
+      );
+
+      alignmentMap[child.name] = targetWorldCoordinates;
+    });
+
+    return alignmentMap;
+  }
+
+  alignCharacterToPose(bones: Bone[], targetPositions: Record<string, Vector3>) {
+    const tempVec1 = new Vector3();
+    const tempVec2 = new Vector3();
+    const quat = new Quaternion();
+
+    bones.forEach(bone => {
+      if (!targetPositions[bone.name]) return;  // Skip if no target is given
+
+      const targetPos = targetPositions[bone.name];
+
+      // Get current world position
+      const currentPos = bone.getWorldPosition(new Vector3());
+
+      // Compute position offset (not always needed for Mixamo rigs)
+      const positionOffset = tempVec1.subVectors(targetPos, currentPos);
+      bone.position.add(positionOffset); // Only apply if necessary
+
+      if (bone.parent && bone.parent.type === "Bone") {
+        // Compute world-space bone direction before and after transformation
+        const childPos = bone.getWorldPosition(new Vector3());
+        const parentPos = bone.parent.getWorldPosition(new Vector3());
+        const parentQuat = bone.parent.getWorldQuaternion(new Quaternion());
+
+        // Compute direction vectors
+        tempVec1.subVectors(childPos, parentPos).normalize(); // Current direction
+        tempVec2.subVectors(targetPos, parentPos).normalize(); // Target direction
+
+        // Compute rotation needed to align current direction to target direction
+        quat.setFromUnitVectors(tempVec1, tempVec2);
+
+        // Convert world-space rotation to local-space
+        const inverseParentQuat = parentQuat.clone().invert();
+        quat.premultiply(inverseParentQuat);
+
+        bone.quaternion.premultiply(quat);
+      }
+    });
+  }
+
+  testRun(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) {
+    const scene = this.editor.timeline.scene;
+    const character = scene.get_object_by_uuid(characterId);
+    if (!character) {
+      console.error("Character not found with id: ", characterId);
+      return;
+    }
+    const alignmentMap = this.getAlignmentMap(character, poseData);
+    const sortedBones = this.getSortedBoneArray(character);
+    this.alignCharacterToPose(sortedBones, alignmentMap);
+  }
+
+  getSortedBoneArray(character: Object3D<Object3DEventMap>) {
+    function getBoneHierarchyDepth(bone: Bone, root: Bone): number {
+      let depth = 0;
+      while (bone !== root && bone.parent) {
+        bone = bone.parent as Bone;
+        depth++;
+      }
+      return depth;
+    }
+
+    const skeletonHelper = new SkeletonHelper(character);
+    const rootBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigHips")!;
+    skeletonHelper.bones.sort((a, b) => getBoneHierarchyDepth(a, rootBone) - getBoneHierarchyDepth(b, rootBone));
+    return skeletonHelper.bones;
   }
 
 }

app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts

@@ -0,0 +1,48 @@
+// A mapping between the model bones names and the Mediapipe output index
+export const MixamoPoseMap: Record<string, number> = {
+  "mixamorigLeftArm": 11,
+  "mixamorigRightArm": 12,
+  "mixamorigLeftForeArm": 13,
+  "mixamorigRightForeArm": 14,
+  "mixamorigLeftHand": 15,
+  "mixamorigRightHand": 16,
+  "mixamorigLeftUpLeg": 23,
+  "mixamorigRightUpLeg": 24,
+  "mixamorigLeftLeg": 25,
+  "mixamorigRightLeg": 26,
+  "mixamorigLeftFoot": 27,
+  "mixamorigRightFoot": 28,
+  "mixamorigLeftHandThumb2": 21,
+  "mixamorigRightHandThumb2": 22,
+}
+
+export const MixamoInterpolationBoneNames = {
+  Spine: "mixamorigSpine",
+  Spine1: "mixamorigSpine1",
+  Spine2: "mixamorigSpine2",
+  Neck: "mixamorigNeck",
+  LeftShoulderBlade: "mixamorigLeftShoulder",
+  DefLeftUpperArm: "DEF-upper_armL001",
+  DefLeftForeArm: "DEF-forearmL001",
+  RightShoulderBlade: "mixamorigRightShoulder",
+  DefRightUpperArm: "DEF-upper_armR001",
+  DefRightForeArm: "DEF-forearmR001",
+  DefLeftThigh: "DEF-thighL001",
+  DefRightThigh: "DEF-thighR001",
+}
+
+export const PoseMap = {
+  NOSE: 0,
+  LEFT_SHOULDER: 11,
+  RIGHT_SHOULDER: 12,
+  LEFT_ELBOW: 13,
+  RIGHT_ELBOW: 14,
+  LEFT_WRIST: 15,
+  RIGHT_WRIST: 16,
+  LEFT_HIP: 23,
+  RIGHT_HIP: 24,
+  LEFT_KNEE: 25,
+  RIGHT_KNEE: 26,
+  LEFT_ANKLE: 27,
+  RIGHT_ANKLE: 28,
+}

app/pages/PageEnigma/Editor/timeline.ts

@@ -273,6 +273,10 @@ export class TimeLine {
       console.debug("Added character object")
       console.debug("object:", obj)
 
+      // Add a skeleton helper to the character
+      const skeletonHelper = new THREE.SkeletonHelper(obj);
+      this.scene.scene.add(skeletonHelper);
+
       const object_uuid = obj.uuid;
 
       this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.

app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx

@@ -1,7 +1,7 @@
 import { faImage, faTrashCan } from "@fortawesome/pro-solid-svg-icons";
 import { ButtonIconStack } from "~/components/reusable/ButtonIconStack";
 import { frameTrackButtonWidthPx } from "../../signals";
-import { useCallback, useContext, useRef, useState } from "react";
+import { useCallback, useContext, useEffect, useRef, useState } from "react";
 import { EngineContext } from "../../contexts/EngineContext";
 import { CHARACTER_FRAME_FILE_TYPE } from "~/enums";
 import { UploadModalMedia } from "~/components/reusable/UploadModalMedia";
@@ -60,6 +60,16 @@ export default function CharacterFrameButton(
     setIsUploadModalOpen(false);
   }, [setIsUploadModalOpen]);
 
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  const handleTest = () => {
+    async function test() {
+      const poseHelper = new CharacterPoseHelper(editorEngine!);
+      const poseData = await poseHelper.extractPoseData("https://cdn-2.fakeyou.com/media/y/6/q/s/v/y6qsvxh66qw9bq8a48619t5pg9sdeyzx/image_y6qsvxh66qw9bq8a48619t5pg9sdeyzx.png")
+      // poseHelper.applyPoseDataToCharacter(characterId, poseData);
+      poseHelper.testRun(characterId, poseData);
+    }
+    test();
+  };
 
   const handleFrameSet = useCallback((token?: string) => {
     if (!token) {
@@ -72,9 +82,9 @@ export default function CharacterFrameButton(
     get_media_url(token)
       .then((url) => {
         console.debug("Frame url: ", url)
+
         // Handle the pose data
-        const poseHelper = new CharacterPoseHelper(editorEngine!);
-        poseHelper.extractPoseData(url);
+        handleTest();
         setMediaFile(url);
       })
       .catch((error) => {
@@ -84,7 +94,7 @@ export default function CharacterFrameButton(
         unlockButton();
       })
 
-  }, [unlockButton, setMediaFile]);
+  }, [unlockButton, setMediaFile, handleTest]);
 
   const handleDeleteFrame = useCallback(() => {
     setMediaFile(undefined);
@@ -96,7 +106,7 @@ export default function CharacterFrameButton(
     return (
       <>
         <div className={className} style={{ minWidth: frameTrackButtonWidthPx, width: frameTrackButtonWidthPx }}>
-          <ButtonIconStack icon={faImage} additionalStyle="bg-character-frame" text={CharacterFrameStrings[target]} onClick={handleFrameClick} />
+          <ButtonIconStack icon={faImage} additionalStyle="bg-character-frame" text={CharacterFrameStrings[target]} onClick={handleTest} />
         </div>
         <UploadImageMediaModal
           isOpen={isUploadModalOpen}