diff --git a/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts b/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts index 4e7a1a9952..841c4f6715 100644 --- a/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts +++ b/app/pages/PageEnigma/Editor/Engines/Helpers/CharacterPoseHelper.ts @@ -1,7 +1,12 @@ import { loadImage } from "~/Helpers/ImageHelpers"; import { EditorExpandedI } from "~/pages/PageEnigma/contexts/EngineContext"; import { FilesetResolver, HandLandmarker, HandLandmarkerResult, PoseLandmarker, PoseLandmarkerResult } from "@mediapipe/tasks-vision" +import { MixamoInterpolationBoneNames, MixamoPoseMap } from "../Mappers/MixamoPoseMapper"; +import { AxesHelper, Bone, Box3, Box3Helper, BufferGeometry, Euler, Matrix4, Object3D, Object3DEventMap, Quaternion, SkeletonHelper, Vector3 } from "three"; +// TODO: Currently the class uses the scene to use as a detachment parent +// Maybe this would cause problems in the future +// Look into an alternative way to detach bone from parent temporarily because reattaching? export class CharacterPoseHelper { editor: EditorExpandedI; @@ -26,7 +31,7 @@ export class CharacterPoseHelper { ); } - async extractPoseData(url: string): { hands: HandLandmarkerResult, pose: PoseLandmarkerResult } { + async extractPoseData(url: string): Promise<{ hands: HandLandmarkerResult; pose: PoseLandmarkerResult; }> { const frameImage = new Image(); frameImage.crossOrigin = "anonymous"; frameImage.src = url; @@ -68,14 +73,291 @@ export class CharacterPoseHelper { applyPoseDataToCharacter(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) { // Apply the pose data to the character - const character = this.editor.timeline.scene.get_object_by_uuid(characterId); + const scene = this.editor.timeline.scene; + const character = scene.get_object_by_uuid(characterId); if (!character) { console.error("Character not found with id: ", characterId); return; } // Apply the pose data to the character - console.debug("Character:", character) + // + // Get character dimensions with a bounding box (unreliable but works for now) + const characterBox = new Box3().setFromObject(character); + + // Add a box helper to the scene + const boxHelper = new Box3Helper(characterBox, 0xfff000); + scene.scene.add(boxHelper); + + // Get the original hip position + const hipPos = new Vector3(); + const hipBone = character.children[0].children.find((child) => child.name === "mixamorigHips")!; + hipBone.getWorldPosition(hipPos); + + // Test only the rotation without pose application for now + // TODO: Remove + this.updateBoneRotations(character); + return; + + // We'll do a triple pass, one that'll gather the bone data + // and another that'll update the main mapped bones and then finally interpolate the extra bones + // + // First pass: Build a list of bones to update + const mappedBones: Record> = {}; + const interpolationBones: Record> = {}; + character.traverse((child) => { + if (child.type !== "Bone") { + return; + } + + if (MixamoPoseMap[child.name] !== undefined) { + console.debug("Mapped bone found:", child.name); + mappedBones[child.name] = child; + } else { + // console.debug("Interpolation bone found:", child.name); + interpolationBones[child.name] = child; + } + }); + + // Second pass: Update the mapped bones + const scale = 1.1; + Object.values(mappedBones).forEach((bone) => { + const mapIndex = MixamoPoseMap[bone.name]; + const poseCoordinates = poseData.pose.worldLandmarks[0][mapIndex]; + const parent = bone.parent!; + + // Detach child from parent + scene.scene.attach(bone); + + bone.position.set(poseCoordinates.x * scale, -poseCoordinates.y * scale, -poseCoordinates.z * scale); + bone.position.add(hipPos); // Pose data world coordinates are centered at center of hips (left and right) + + // Reattach child to its parent + parent.attach(bone); + }); + + // Third pass: Interpolate the extra bones + if (interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh]) { + // Left thigh is lipo'd between left upper leg and knee + const leftThigh = interpolationBones[MixamoInterpolationBoneNames.DefLeftThigh]; + const leftUpperLeg = mappedBones["mixamorigLeftUpLeg"]; + const leftKnee = mappedBones["mixamorigLeftLeg"]; + const parent = leftThigh.parent!; + + // Detach child from parent + scene.scene.attach(leftThigh); + + // Calculate the lipo position + const leftUpperLegPos = new Vector3(); + const leftKneePos = new Vector3(); + leftUpperLeg.getWorldPosition(leftUpperLegPos); + console.debug("Left upper leg pos: ", leftUpperLegPos); + leftKnee.getWorldPosition(leftKneePos); + console.debug("Left knee pos: ", leftKneePos); + const leftThighPos = new Vector3(); + leftThighPos.lerpVectors(leftUpperLegPos, leftKneePos, 0.6); + console.debug("Left thigh pos: ", leftThighPos); + leftThigh.position.set(leftThighPos.x, leftThighPos.y, leftThighPos.z); + + // Reattach child to its parent + parent.attach(leftThigh); + } + + if (interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm]) { + // Right upper arm is lipo'd between shoulder and elbow + const rightUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefRightUpperArm]; + const rightShoulder = mappedBones["mixamorigRightArm"]; + const rightElbow = mappedBones["mixamorigRightForeArm"]; + const parent = rightUpperArm.parent!; + + // Detach child from parent + scene.scene.attach(rightUpperArm); + + // Calculate the lipo position + const rightShoulderPos = new Vector3(); + const rightElbowPos = new Vector3(); + rightShoulder.getWorldPosition(rightShoulderPos); + console.debug("Right shoulder pos: ", rightShoulderPos); + rightElbow.getWorldPosition(rightElbowPos); + console.debug("Right elbow pos: ", rightElbowPos); + const rightUpperArmPos = new Vector3(); + rightUpperArmPos.lerpVectors(rightShoulderPos, rightElbowPos, 0.5); + console.debug("Right upper arm pos: ", rightUpperArmPos); + rightUpperArm.position.set(rightUpperArmPos.x, rightUpperArmPos.y, rightUpperArmPos.z); + + // Reattach child to its parent + parent.attach(rightUpperArm); + } + + if (interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm]) { + // Left upper arm is lipo'd between shoulder and elbow + const leftUpperArm = interpolationBones[MixamoInterpolationBoneNames.DefLeftUpperArm]; + console.log("Left def arm position:", leftUpperArm.position); + const leftShoulder = mappedBones["mixamorigLeftArm"]; + const leftElbow = mappedBones["mixamorigLeftForeArm"]; + const parent = leftUpperArm.parent!; + + // Detach child from parent + scene.scene.attach(leftUpperArm); + + // Calculate the lipo position + const leftShoulderPos = new Vector3(); + const leftElbowPos = new Vector3(); + leftShoulder.getWorldPosition(leftShoulderPos); + console.debug("Left shoulder pos: ", leftShoulderPos); + leftElbow.getWorldPosition(leftElbowPos); + console.debug("Left elbow pos: ", leftElbowPos); + const leftUpperArmPos = new Vector3(); + leftUpperArmPos.lerpVectors(leftShoulderPos, leftElbowPos, 0.5); + console.debug("Left upper arm pos: ", leftUpperArmPos); + leftUpperArm.position.set(leftUpperArmPos.x, leftUpperArmPos.y, leftUpperArmPos.z); + + // Reattach child to its parent + parent.attach(leftUpperArm); + console.log("Left def arm position:", leftUpperArm.position); + } + + // this.updateBoneRotations(character); + } + + updateBoneRotations(character: Object3D) { + const skeletonHelper = new SkeletonHelper(character); + const rootBone = skeletonHelper.bones[0]; + const queue = [rootBone]; + const scene = this.editor.timeline.scene; + + const fromBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftArm")!; + const toBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigLeftForeArm")!; + const fromHelper = new AxesHelper(10); + const toHelper = new AxesHelper(10); + fromHelper.material.transparent = true; + fromHelper.material.depthTest = false; + fromBone.add(fromHelper); + toBone.add(toHelper); + + console.debug("Before rotations: ", fromBone, toBone); + this.updateBoneRotationEuler(fromBone, toBone); + console.debug("After rotations: ", fromBone, toBone); + } + + updateBoneRotationEuler(bone: Bone, childBone: Bone) { + // Get world positions + const boneWorldPos = new Vector3(); + const childWorldPos = new Vector3(); + bone.getWorldPosition(boneWorldPos); + childBone.getWorldPosition(childWorldPos); + + // Compute the new direction the bone should point to + const targetDirection = new Vector3().subVectors(childWorldPos, boneWorldPos).normalize(); + console.debug("Target direction: ", targetDirection); + + // Convert world direction to local space (relative to bone) + const parentMatrix = new Matrix4(); + if (bone.parent) { + bone.parent.updateMatrixWorld(true); + parentMatrix.copy(bone.matrixWorld).invert(); // Convert to local space + } + targetDirection.applyMatrix4(parentMatrix); + console.debug("Target direction local: ", targetDirection); + + // Compute Euler angles from the target direction + const eulerRotation = new Euler(); + eulerRotation.setFromVector3(targetDirection); + + // Apply rotation + bone.rotation.copy(eulerRotation); + bone.updateMatrixWorld(true); + } + + getAlignmentMap(character: Object3D, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) { + const alignmentMap: Record = {}; + + character.traverse((child) => { + if (child.type !== "Bone") { + return; + } + + if (!MixamoPoseMap[child.name]) { + return; + } + + const mapIndex = MixamoPoseMap[child.name]; + const targetCoordinates = poseData.pose.worldLandmarks[0][mapIndex]; + const targetWorldCoordinates = new Vector3( + targetCoordinates.x, -targetCoordinates.y, -targetCoordinates.z + ); + + alignmentMap[child.name] = targetWorldCoordinates; + }); + + return alignmentMap; + } + + alignCharacterToPose(bones: Bone[], targetPositions: Record) { + const tempVec1 = new Vector3(); + const tempVec2 = new Vector3(); + const quat = new Quaternion(); + + bones.forEach(bone => { + if (!targetPositions[bone.name]) return; // Skip if no target is given + + const targetPos = targetPositions[bone.name]; + + // Get current world position + const currentPos = bone.getWorldPosition(new Vector3()); + + // Compute position offset (not always needed for Mixamo rigs) + const positionOffset = tempVec1.subVectors(targetPos, currentPos); + bone.position.add(positionOffset); // Only apply if necessary + + if (bone.parent && bone.parent.type === "Bone") { + // Compute world-space bone direction before and after transformation + const childPos = bone.getWorldPosition(new Vector3()); + const parentPos = bone.parent.getWorldPosition(new Vector3()); + const parentQuat = bone.parent.getWorldQuaternion(new Quaternion()); + + // Compute direction vectors + tempVec1.subVectors(childPos, parentPos).normalize(); // Current direction + tempVec2.subVectors(targetPos, parentPos).normalize(); // Target direction + + // Compute rotation needed to align current direction to target direction + quat.setFromUnitVectors(tempVec1, tempVec2); + + // Convert world-space rotation to local-space + const inverseParentQuat = parentQuat.clone().invert(); + quat.premultiply(inverseParentQuat); + + bone.quaternion.premultiply(quat); + } + }); + } + + testRun(characterId: string, poseData: { hands: HandLandmarkerResult, pose: PoseLandmarkerResult }) { + const scene = this.editor.timeline.scene; + const character = scene.get_object_by_uuid(characterId); + if (!character) { + console.error("Character not found with id: ", characterId); + return; + } + const alignmentMap = this.getAlignmentMap(character, poseData); + const sortedBones = this.getSortedBoneArray(character); + this.alignCharacterToPose(sortedBones, alignmentMap); + } + + getSortedBoneArray(character: Object3D) { + function getBoneHierarchyDepth(bone: Bone, root: Bone): number { + let depth = 0; + while (bone !== root && bone.parent) { + bone = bone.parent as Bone; + depth++; + } + return depth; + } + + const skeletonHelper = new SkeletonHelper(character); + const rootBone = skeletonHelper.bones.find((bone) => bone.name === "mixamorigHips")!; + skeletonHelper.bones.sort((a, b) => getBoneHierarchyDepth(a, rootBone) - getBoneHierarchyDepth(b, rootBone)); + return skeletonHelper.bones; } } diff --git a/app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts b/app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts new file mode 100644 index 0000000000..e1749f6cd3 --- /dev/null +++ b/app/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts @@ -0,0 +1,48 @@ +// A mapping between the model bones names and the Mediapipe output index +export const MixamoPoseMap: Record = { + "mixamorigLeftArm": 11, + "mixamorigRightArm": 12, + "mixamorigLeftForeArm": 13, + "mixamorigRightForeArm": 14, + "mixamorigLeftHand": 15, + "mixamorigRightHand": 16, + "mixamorigLeftUpLeg": 23, + "mixamorigRightUpLeg": 24, + "mixamorigLeftLeg": 25, + "mixamorigRightLeg": 26, + "mixamorigLeftFoot": 27, + "mixamorigRightFoot": 28, + "mixamorigLeftHandThumb2": 21, + "mixamorigRightHandThumb2": 22, +} + +export const MixamoInterpolationBoneNames = { + Spine: "mixamorigSpine", + Spine1: "mixamorigSpine1", + Spine2: "mixamorigSpine2", + Neck: "mixamorigNeck", + LeftShoulderBlade: "mixamorigLeftShoulder", + DefLeftUpperArm: "DEF-upper_armL001", + DefLeftForeArm: "DEF-forearmL001", + RightShoulderBlade: "mixamorigRightShoulder", + DefRightUpperArm: "DEF-upper_armR001", + DefRightForeArm: "DEF-forearmR001", + DefLeftThigh: "DEF-thighL001", + DefRightThigh: "DEF-thighR001", +} + +export const PoseMap = { + NOSE: 0, + LEFT_SHOULDER: 11, + RIGHT_SHOULDER: 12, + LEFT_ELBOW: 13, + RIGHT_ELBOW: 14, + LEFT_WRIST: 15, + RIGHT_WRIST: 16, + LEFT_HIP: 23, + RIGHT_HIP: 24, + LEFT_KNEE: 25, + RIGHT_KNEE: 26, + LEFT_ANKLE: 27, + RIGHT_ANKLE: 28, +} diff --git a/app/pages/PageEnigma/Editor/timeline.ts b/app/pages/PageEnigma/Editor/timeline.ts index 356e123097..4036da9036 100644 --- a/app/pages/PageEnigma/Editor/timeline.ts +++ b/app/pages/PageEnigma/Editor/timeline.ts @@ -273,6 +273,10 @@ export class TimeLine { console.debug("Added character object") console.debug("object:", obj) + // Add a skeleton helper to the character + const skeletonHelper = new THREE.SkeletonHelper(obj); + this.scene.scene.add(skeletonHelper); + const object_uuid = obj.uuid; this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character. diff --git a/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx b/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx index 7c25976bfc..59bb24a0dd 100644 --- a/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx +++ b/app/pages/PageEnigma/comps/Timeline/CharacterFrameButtons.tsx @@ -1,7 +1,7 @@ import { faImage, faTrashCan } from "@fortawesome/pro-solid-svg-icons"; import { ButtonIconStack } from "~/components/reusable/ButtonIconStack"; import { frameTrackButtonWidthPx } from "../../signals"; -import { useCallback, useContext, useRef, useState } from "react"; +import { useCallback, useContext, useEffect, useRef, useState } from "react"; import { EngineContext } from "../../contexts/EngineContext"; import { CHARACTER_FRAME_FILE_TYPE } from "~/enums"; import { UploadModalMedia } from "~/components/reusable/UploadModalMedia"; @@ -60,6 +60,16 @@ export default function CharacterFrameButton( setIsUploadModalOpen(false); }, [setIsUploadModalOpen]); + // eslint-disable-next-line react-hooks/exhaustive-deps + const handleTest = () => { + async function test() { + const poseHelper = new CharacterPoseHelper(editorEngine!); + const poseData = await poseHelper.extractPoseData("https://cdn-2.fakeyou.com/media/y/6/q/s/v/y6qsvxh66qw9bq8a48619t5pg9sdeyzx/image_y6qsvxh66qw9bq8a48619t5pg9sdeyzx.png") + // poseHelper.applyPoseDataToCharacter(characterId, poseData); + poseHelper.testRun(characterId, poseData); + } + test(); + }; const handleFrameSet = useCallback((token?: string) => { if (!token) { @@ -72,9 +82,9 @@ export default function CharacterFrameButton( get_media_url(token) .then((url) => { console.debug("Frame url: ", url) + // Handle the pose data - const poseHelper = new CharacterPoseHelper(editorEngine!); - poseHelper.extractPoseData(url); + handleTest(); setMediaFile(url); }) .catch((error) => { @@ -84,7 +94,7 @@ export default function CharacterFrameButton( unlockButton(); }) - }, [unlockButton, setMediaFile]); + }, [unlockButton, setMediaFile, handleTest]); const handleDeleteFrame = useCallback(() => { setMediaFile(undefined); @@ -96,7 +106,7 @@ export default function CharacterFrameButton( return ( <>
- +