From 1351a40b3d84f97fbefeb96214cd48059ba8a422 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Tue, 28 Apr 2026 21:56:58 -0700 Subject: [PATCH] PageEnigma refactor checkpoint: delete timeline subsystem and add Zustand store Phase 1-3 of the 3D page refactor: - Delete the timeline + event-queue subsystem (Queue/, useQueueHandler, Editor/timeline.ts, Editor/Engines/, clips/, and the timeline-only signal slices: timeline, timing, audioGroup, cameraGroup, promptTravelGroup, utils/publishClip) - Delete dead-code files with no live importers: lipsync.js, retargeting.ts, preview_video_generation.ts, shaders.js, placeholder_manager.ts, both composable_engines/*.tsx stubs, contexts/TrackContext/, comps/TimerGrid, comps/ControlsVideo - Add PageEnigmaStore.ts with the full state shape that will replace preact-signals (scene objects, cameras, transform mode, layout, error overlays, generation/stylize state) - Strip signals/index.ts of deleted slice exports and reduce signals/sizing.ts to just sidePanelWidth Branch does not yet compile -- subsequent commits will strip Queue/timeline references from editor.ts, the remaining signal files, and component consumers, then build out actions/ and event hooks. --- .../Engines/CharacterAnimationEngine.ts | 308 ------ .../Engines/Mappers/MixamoPoseMapper.ts | 48 - .../Editor/Engines/animation_engine.ts | 27 - .../PageEnigma/Editor/Engines/audio_engine.ts | 81 -- .../Editor/Engines/emotion_engine.ts | 24 - .../Editor/Engines/lip_sync_engine.ts | 22 - .../Editor/Engines/transform_engine.ts | 58 -- .../src/pages/PageEnigma/Editor/lipsync.js | 239 ----- .../PageEnigma/Editor/placeholder_manager.ts | 117 --- .../Editor/preview_video_generation.ts | 41 - .../pages/PageEnigma/Editor/retargeting.ts | 97 -- .../src/pages/PageEnigma/Editor/shaders.js | 52 - .../src/pages/PageEnigma/Editor/timeline.ts | 950 ------------------ .../src/pages/PageEnigma/PageEnigmaStore.ts | 394 ++++++++ .../app/src/pages/PageEnigma/Queue/Queue.ts | 127 --- .../src/pages/PageEnigma/Queue/QueueNames.ts | 5 - .../PageEnigma/Queue/fromEngineActions.ts | 14 - .../app/src/pages/PageEnigma/Queue/index.ts | 3 - .../pages/PageEnigma/Queue/toEngineActions.ts | 47 - .../PageEnigma/Queue/toInferenceActions.ts | 3 - .../PageEnigma/Queue/toTimelineActions.ts | 4 - .../clips/ClipInterfaces/IAnimationClip.ts | 7 - .../pages/PageEnigma/clips/animation_clip.ts | 348 ------- .../src/pages/PageEnigma/clips/audio_clip.ts | 69 -- .../app/src/pages/PageEnigma/clips/clip_ui.ts | 71 -- .../pages/PageEnigma/clips/emotion_clip.ts | 143 --- .../pages/PageEnigma/clips/lipsync_clip.ts | 171 ---- .../pages/PageEnigma/clips/transform_clip.ts | 238 ----- .../composable_engines/audio_engine.tsx | 0 .../composable_engines/lipsync_engine.tsx | 0 .../comps/ControlsVideo/ControlsVideo.tsx | 158 --- .../PageEnigma/comps/ControlsVideo/index.tsx | 1 - .../PageEnigma/comps/TimerGrid/TimerGrid.tsx | 84 -- .../contexts/TrackContext/TrackContext.tsx | 11 - .../contexts/TrackContext/TrackProvider.tsx | 27 - .../TrackContext/utils/useUpdateDragDrop.ts | 12 - .../pages/PageEnigma/hooks/useQueueHandler.ts | 126 --- .../pages/PageEnigma/signals/audioGroup.ts | 152 --- .../pages/PageEnigma/signals/cameraGroup.ts | 148 --- .../app/src/pages/PageEnigma/signals/index.ts | 4 - .../PageEnigma/signals/promptTravelGroup.ts | 120 --- .../src/pages/PageEnigma/signals/sizing.ts | 64 +- .../src/pages/PageEnigma/signals/timeline.ts | 68 -- .../src/pages/PageEnigma/signals/timing.ts | 12 - .../PageEnigma/signals/utils/publishClip.ts | 25 - 45 files changed, 395 insertions(+), 4325 deletions(-) delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/CharacterAnimationEngine.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/animation_engine.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/audio_engine.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/emotion_engine.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/lip_sync_engine.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/transform_engine.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/lipsync.js delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/placeholder_manager.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/preview_video_generation.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/retargeting.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/shaders.js delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/timeline.ts create mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/PageEnigmaStore.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/Queue.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/QueueNames.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/fromEngineActions.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/index.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toEngineActions.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toInferenceActions.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toTimelineActions.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/ClipInterfaces/IAnimationClip.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/animation_clip.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/audio_clip.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/clip_ui.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/emotion_clip.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/lipsync_clip.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/clips/transform_clip.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/composable_engines/audio_engine.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/composable_engines/lipsync_engine.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/index.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/comps/TimerGrid/TimerGrid.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/hooks/useQueueHandler.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/signals/audioGroup.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/signals/cameraGroup.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/signals/promptTravelGroup.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timeline.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timing.ts delete mode 100644 frontend/apps/artcraft/app/src/pages/PageEnigma/signals/utils/publishClip.ts diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/CharacterAnimationEngine.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/CharacterAnimationEngine.ts deleted file mode 100644 index 83015964ad..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/CharacterAnimationEngine.ts +++ /dev/null @@ -1,308 +0,0 @@ -import * as THREE from "three"; -import { Retarget } from "../retargeting"; -import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js"; -import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; -import { get_media_url } from "~/Classes/ApiHelpers"; -import { MMDLoader } from "three/examples/jsm/loaders/MMDLoader.js"; -import { ClipUI } from "../../clips/clip_ui"; -import Ijson from "~/interfaces/Ijson"; - -export const START_FRAME_LABEL = "CharacterStartFrameLabel"; - -export class CharacterAnimationEngine implements Ijson { - version: number; - characterAnimations: Map, ClipUI[]> = new Map(); - characterMixers = new Map, THREE.AnimationMixer>(); - - constructor(version: number) { - this.version = version; - } - - toJSON(): unknown { - // Return just an array of character object IDs - const characterIds = Array.from(this.characterAnimations.keys()).map((character) => character.uuid); - return { characters: characterIds }; - } - - /** - * A single character model is capable of tracking multiple animations, in its own track - * These tracks can be blended with weights or additively - * The weight change can be done via custom interpolation logic - * - * @param objectUUID The UUID of the object. - * @param clip The 3JS animation clip - * @param clipUI The clip info - */ - addCharacterAnimation(character: THREE.Object3D, clip: THREE.AnimationClip, clipUI: ClipUI) { - // If character doesn't exist, create a clip object - // If it does exist, add a new animation to the existing character by extracting it - // TODO: FIXME: Duplicate prevention? - clip.name = clipUI.media_id; - character.animations.push(clip); - console.log("Adding animation track:") - console.log(clip); - - if (!this.characterAnimations.has(character)) { - this.characterAnimations.set(character, []); - this.#ensureMixerExists(character); - } - - // Add the clip info to the character map and sort the clips - const characterClips = this.characterAnimations.get(character)!; - characterClips.push(clipUI); - characterClips.sort((a, b) => a.offset - b.offset); - } - - /** - * @param objectUUID The UUID of the object. - * @param mediaId The media ID of the animation. - * @param clipUI The clip info - */ - async addCharacterAnimationMedia(character: THREE.Object3D, mediaId: string, clipUI: ClipUI) { - // If character doesn't exist, create a clip object - // If it does exist, add a new animation to the existing character by extracting it - // TODO: FIXME: Duplicate prevention? - const animationTrack = await this.#load_animation(character, mediaId); - this.addCharacterAnimation(character, animationTrack, clipUI); - } - - #ensureMixerExists(character: THREE.Object3D) { - if (!this.characterMixers.has(character)) { - this.characterMixers.set(character, new THREE.AnimationMixer(character)); - } - } - - getMixer(character: THREE.Object3D) { - this.#ensureMixerExists(character); - return this.characterMixers.get(character)!; - } - - #interpolateClips(character: THREE.Object3D, timestamp: number, maxTime: number) { - const clips = this.characterAnimations.get(character); - - const mixer = this.getMixer(character); - - // Find the two clips we're interpolating between - let prevClip: ClipUI | null = null; - let nextClip: ClipUI | null = null; - const firstClip = clips ? clips[0] : null; - const lastClip = clips ? clips[clips.length - 1] : null; - - // If we're before the first clip, interpolate it in from base pose - // TODO: Replace with starting frame - if (firstClip && firstClip.offset > timestamp) { - prevClip = null; - nextClip = firstClip; - } else if (lastClip && (lastClip.offset + lastClip.length) < timestamp) { // After last clip, interpolate it out - prevClip = lastClip; - nextClip = null; - } else if (clips) { - let prevClipIndex = 0; - - // Loop until the next clip is after the timestamp - while (prevClipIndex < clips.length - 1 && clips[prevClipIndex + 1].offset < timestamp) { - prevClipIndex++; - } - - prevClip = clips[prevClipIndex]; - nextClip = clips[prevClipIndex + 1]; - } - - // If the clips exist, fetch their actions. If not, evaluation is either at the start or the end. - // If evalution is on either sides, check the start/end frame tracks - // Otherwise just mark it null - const prevAnimationTrack = prevClip ? - this.getCharacterAnimationTrack(character, prevClip.media_id) : - this.getCharacterAnimationTrack(character, START_FRAME_LABEL); - - // TODO: Replace with ending frame - const nextAnimationTrack = nextClip ? this.getCharacterAnimationTrack(character, nextClip.media_id) : null; - - const prevAction = prevAnimationTrack ? mixer.clipAction(prevAnimationTrack) : null; - const nextAction = nextAnimationTrack ? mixer.clipAction(nextAnimationTrack) : null; - - // Calculate the progress of timestamp from end of prev action to start of next action - const left = (prevClip?.offset ?? 0) + (prevClip?.length ?? 0); - const right = nextClip?.offset ?? maxTime; - - // Simple Linear interpolation for now - // TODO: Take an interpolation dependency, or better yet, write a transition engine - const progress = (timestamp - left) / (right - left); - prevAction?.setEffectiveWeight(1 - progress); - nextAction?.setEffectiveWeight(progress); - - // Make sure we hold that last frame for the previous action - if (prevAction) { - prevAction.clampWhenFinished = true; - } - - // The next action should stay at the first frame - if (nextAction) { - nextAction.paused = true; - } - - // Necessary to ensure the actions are active - the default is inactive, mixer won't do anything - prevAction?.play(); - nextAction?.play(); - - // The clip time would still be relative to the previous clip - const clipTime = timestamp - (prevClip?.offset ?? 0); - mixer.setTime(clipTime / 1000); - } - - evaluateCharacter(character: THREE.Object3D, timestamp: number, maxTime: number) { - const mixer = this.getMixer(character); - const clips = this.characterAnimations.get(character); - - // Find the clip we're in right now - const currentClip = clips?.find((clip) => { - return clip.offset <= timestamp && clip.offset + clip.length >= timestamp; - }); - - // If timestamp not in any clip, do nothing. - // If timestamp in clip, set mixer to the timestamp inside the clip - if (!currentClip) { - // Let the interpolation function handle this actions - this.#interpolateClips(character, timestamp, maxTime) - return; - } - - const clipTime = timestamp - currentClip.offset; - const animationTrack = this.getCharacterAnimationTrack(character, currentClip.media_id); - - // NOTE: This shouldn't really happen unless the UI was desynced at some point from the engine - if (!animationTrack) { - return; - } - - // Play only this action - const animationAction = mixer.clipAction(animationTrack); - - // Since it's the only clip in this timestamp, make it full weight and make sure it's not paused (from interpolation or otherwise) - animationAction.setEffectiveWeight(1); - animationAction.paused = false; - - // Necessary to ensure the actions are active - the default is inactive, mixer won't do anything - animationAction.play(); - - mixer.setTime(clipTime / 1000); - } - - getCharacterAnimationTrack(character: THREE.Object3D, mediaId: string) { - return character.animations.find((clip) => clip.name === mediaId); - } - - /** Evaluate all character animations at a given timestamp (milliseconds) */ - evaluate(timestamp: number, maxTime: number) { - this.characterMixers.forEach((_, character) => { - this.evaluateCharacter(character, timestamp, maxTime); - }) - } - - stopCharacter(character: THREE.Object3D) { - const mixer = this.characterMixers.get(character); - console.debug("Stopping all actions for character", character); - console.debug(mixer); - mixer?.stopAllAction(); - console.debug(mixer); - } - - stop() { - this.characterMixers.forEach((mixer) => { - mixer.stopAllAction(); - }); - } - - async #load_animation(character: THREE.Object3D, mediaId: string): Promise { - // Get the file URL and extract the (first) animation track from it - // TODO: Support for multiple animations in a single file? - const url = await get_media_url(mediaId); - - return await new Promise((resolve) => { - if (url.includes(".glb")) { - const glbLoader = new GLTFLoader(); - - glbLoader.load(url, (gltf) => { - const animationClip = gltf.animations[0]; - resolve(animationClip); - }); - } else if (url.includes(".fbx")) { - const fbxLoader = new FBXLoader(); - fbxLoader.load(url, (fbx) => { - const animationClip_1 = fbx.animations[0]; - animationClip_1.tracks.forEach((track) => { - const retarget = new Retarget(); - const retarget_value = retarget.retarget(track.name); - track.name = retarget_value.bone; - console.log(track); - if (retarget_value.is_special) { - // TODO: Revisit special properties later - // this.special_properties.push(retarget_value); - } - }); - resolve(animationClip_1); - }); - } else if (url.includes(".vmd")) { - const mmdLoader = new MMDLoader(); - mmdLoader.loadAnimation(url, character as THREE.SkinnedMesh, (mmd) => { - mmd.name = url; - resolve(mmd as THREE.AnimationClip); - }); - } - }); - } - - removeCharacter(character: THREE.Object3D) { - console.debug("Removing character from animation engine", character); - this.characterAnimations.delete(character); - this.characterMixers.delete(character); - } - - removeAnimation(character: THREE.Object3D, clip: ClipUI) { - console.debug("Removing animation from character", character, clip); - const clips = this.characterAnimations.get(character)!; - - const clipIndex = clips.findIndex((c) => c.clip_uuid === clip.clip_uuid); - if (clipIndex < 0) { - return; - } - - // Remove the clip from character's tracked clips - clips.splice(clipIndex, 1); - - // Stop all character animations to cleanly reset - this.stopCharacter(character); - } - - clearStartFrame(character: THREE.Object3D, resetTime?: number, maxTime = 1) { - // Stop all character animations - this.stopCharacter(character); - - // Check the animations and remove any labelled START_FRAME_LABEL - character.animations = character.animations.filter((clip) => clip.name !== START_FRAME_LABEL); - - // If resetTime is passed, evaluate the character to that time - if (resetTime) { - this.evaluateCharacter(character, resetTime, maxTime); - } - } - - /** resetTime: The time to reset the character to. This should ideally be the current timeline time. */ - async createStartFrameAnimation(character: THREE.Object3D, poseHelper: any, url: string, resetTime: number = 0, maxTime: number = 1) { - //// Clear any existing start frame - //// This will also reset the animation to default pose - //this.clearStartFrame(character); - - //// We must create the track data now after resetting to default - //const poseData = await poseHelper.extractPoseData(url); - //const tracks = poseHelper.inflatePoseDataToTracks(character, poseData); - - //// Create the animation track from the rotation values and hip position - //// then add it to the character - //const startFrameClip = new THREE.AnimationClip(START_FRAME_LABEL, 0, tracks); - //character.animations.push(startFrameClip); - - //// Reset the character evaluation to what was passed in - //this.evaluateCharacter(character, resetTime, maxTime); - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts deleted file mode 100644 index e1749f6cd3..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/Mappers/MixamoPoseMapper.ts +++ /dev/null @@ -1,48 +0,0 @@ -// A mapping between the model bones names and the Mediapipe output index -export const MixamoPoseMap: Record = { - "mixamorigLeftArm": 11, - "mixamorigRightArm": 12, - "mixamorigLeftForeArm": 13, - "mixamorigRightForeArm": 14, - "mixamorigLeftHand": 15, - "mixamorigRightHand": 16, - "mixamorigLeftUpLeg": 23, - "mixamorigRightUpLeg": 24, - "mixamorigLeftLeg": 25, - "mixamorigRightLeg": 26, - "mixamorigLeftFoot": 27, - "mixamorigRightFoot": 28, - "mixamorigLeftHandThumb2": 21, - "mixamorigRightHandThumb2": 22, -} - -export const MixamoInterpolationBoneNames = { - Spine: "mixamorigSpine", - Spine1: "mixamorigSpine1", - Spine2: "mixamorigSpine2", - Neck: "mixamorigNeck", - LeftShoulderBlade: "mixamorigLeftShoulder", - DefLeftUpperArm: "DEF-upper_armL001", - DefLeftForeArm: "DEF-forearmL001", - RightShoulderBlade: "mixamorigRightShoulder", - DefRightUpperArm: "DEF-upper_armR001", - DefRightForeArm: "DEF-forearmR001", - DefLeftThigh: "DEF-thighL001", - DefRightThigh: "DEF-thighR001", -} - -export const PoseMap = { - NOSE: 0, - LEFT_SHOULDER: 11, - RIGHT_SHOULDER: 12, - LEFT_ELBOW: 13, - RIGHT_ELBOW: 14, - LEFT_WRIST: 15, - RIGHT_WRIST: 16, - LEFT_HIP: 23, - RIGHT_HIP: 24, - LEFT_KNEE: 25, - RIGHT_KNEE: 26, - LEFT_ANKLE: 27, - RIGHT_ANKLE: 28, -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/animation_engine.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/animation_engine.ts deleted file mode 100644 index f735250d8e..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/animation_engine.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { AnimationClip } from "../../clips/animation_clip"; - -export class AnimationEngine { - clips: { [key: string]: AnimationClip } = {}; - version: number; - - constructor(version: number) { - this.clips = {}; - this.version = version; - } - - load_object(object_uuid: string, media_id: string, clip_name: string) { - console.debug("load_object", object_uuid, media_id, clip_name); - this.clips[object_uuid + media_id] = new AnimationClip( - this.version, - media_id, - "glb", - object_uuid, - 1.0, - 1.0, - clip_name, - 0.0, - 0.0, - ); - //this.clips[object_uuid + media_id]._load_animation(); - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/audio_engine.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/audio_engine.ts deleted file mode 100644 index 0aecc1ebbd..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/audio_engine.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { AudioClip } from "../../clips/audio_clip"; - -class AudioEngine { - clips: { [key: string]: AudioClip } = {}; - audio_sources: { [key: string]: AudioBufferSourceNode } = {}; - version: number; - playing: string[]; - last_frame: number; - - constructor() { - this.clips = {}; - this.playing = []; - this.audio_sources = {}; - this.version = 1.0; - this.last_frame = 0; - } - - // loads clips into the engine to cache - loadClip(audio_media_id: string) { - if (this.clips[audio_media_id] != null) { - return; - } - this.clips[audio_media_id] = new AudioClip( - this.version, - audio_media_id, - 1.0, - ); - } - - // plays from the timeline. - playClip(audio_media_id: string) { - if (this.clips[audio_media_id] == null) { - this.loadClip(audio_media_id); - } - } - - async step(audio_media_id: string, frame: number, offset: number) { - const frameBuffer = 12; - - const pos = frame - offset; - - let doPlay = true; - if ( - Math.abs(this.last_frame - frame) < frameBuffer - 1 && - this.last_frame !== 0 - ) { - doPlay = false; - } - if (pos <= 1) { - doPlay = true; - } - - const clip = this.clips[audio_media_id]; - - if (doPlay && clip.audio_data?.audioContext) { - const startTime = pos / 60.0; - const endTime = frameBuffer / 60.0; - clip.audio_data.source = - clip.audio_data.audioContext.createBufferSource(); - clip.audio_data.source.buffer = clip.audio_data.audioBuffer; - clip.audio_data.source.connect(clip.audio_data.audioContext.destination); - clip.audio_data.source.start(0, startTime, endTime); - this.last_frame = frame; - } - } - - // stops the clips - stopClip(audio_media_id: string) { - const clip = this.clips[audio_media_id]; - if (clip.audio_data?.source) { - clip.audio_data.source.stop(); - clip.audio_data.source.disconnect(); - } else { - console.warn( - `AudioManager: AudioClip with id "${audio_media_id}" not found.`, - ); - } - } -} - -export default AudioEngine; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/emotion_engine.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/emotion_engine.ts deleted file mode 100644 index afef730f69..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/emotion_engine.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { EmotionClip } from "../../clips/emotion_clip"; - -class EmotionEngine { - clips: { [key: string]: EmotionClip } = {}; - version: number; - - constructor(version: number) { - this.clips = {}; - this.version = version; - } - - // loads clips into the engine to cache - loadClip(object_uuid: string, csv_media_id: string) { - if (this.clips[object_uuid + csv_media_id] != null) { - return; - } - this.clips[object_uuid + csv_media_id] = new EmotionClip( - this.version, - csv_media_id, - ); - } -} - -export default EmotionEngine; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/lip_sync_engine.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/lip_sync_engine.ts deleted file mode 100644 index 8f43d6f951..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/lip_sync_engine.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { LipSyncClip } from "../../clips/lipsync_clip"; - -// needs to be apart of the editor window. -export default class LipSyncEngine { - clips: { [key: string]: LipSyncClip } = {}; - audio_sources: { [key: string]: AudioBufferSourceNode } = {}; - version: number; - - constructor() { - this.clips = {}; - this.audio_sources = {}; - this.version = 1.0; - } - - load_object(object_uuid: string, audio_media_id: string) { - this.clips[object_uuid + audio_media_id] = new LipSyncClip( - this.version, - audio_media_id, - 1.0, - ); - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/transform_engine.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/transform_engine.ts deleted file mode 100644 index 160b38d17a..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/Engines/transform_engine.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { TransformClip } from "../../clips/transform_clip"; -import * as THREE from "three"; - -class TransformEngine { - clips: { [key: string]: TransformClip } = {}; - version: number; - - constructor(version: number) { - this.version = version; - this.clips = {}; - } - - loadObject(object_uuid: string, clip_length: number = 2) { - this.clips[object_uuid] = new TransformClip( - this.version, - object_uuid, - clip_length, - object_uuid, - ); // replace the last object_uuid with the media ID when its ready. - return this.clips[object_uuid]; - } - - addFrame( - object_uuid: string, - clip_length: number = 2, - position: THREE.Vector3, - rotation: THREE.Vector3, - scale: THREE.Vector3, - offset: number, - keyframe_uuid: string, - ): boolean { - if (this.clips[object_uuid] == null) { - this.clips[object_uuid] = new TransformClip( - this.version, - object_uuid, - clip_length, - ); - this.clips[object_uuid].add_frame( - position, - rotation, - scale, - offset, - keyframe_uuid, - ); - return false; - } - this.clips[object_uuid].add_frame( - position, - rotation, - scale, - offset, - keyframe_uuid, - ); - return true; - } -} - -export default TransformEngine; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/lipsync.js b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/lipsync.js deleted file mode 100644 index a0b59ea522..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/lipsync.js +++ /dev/null @@ -1,239 +0,0 @@ -const BoundingFrequencyMasc = [0, 400, 560, 2400, 4800]; -const BoundingFrequencyFem = [0, 500, 700, 3000, 6000]; -const IndicesFrequencyFemale = []; -const IndicesFrequencyMale = []; -const FFT_SIZE = 1024; -const samplingFrequency = 44100; - -for (let m = 0; m < BoundingFrequencyMasc.length; m++) { - IndicesFrequencyMale[m] = Math.round( - ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m], - ); -} - -for (let m = 0; m < BoundingFrequencyFem.length; m++) { - IndicesFrequencyFemale[m] = Math.round( - ((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m], - ); -} - -export class LipSync { - constructor(face) { - this.face = face; - this.buffer = null; - this.last_frame = 0; - - // WE NEED THIS TO be run at RUNTIME - // we have to modify this for our - // this.ee = this.face.morphTargetDictionary["E"]; - // this.oh = this.face.morphTargetDictionary["O"]; - // this.ah = this.face.morphTargetDictionary["aa"]; - } - - // for streaming sound? - start(stream) { - this.audioContext = new AudioContext(); - this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream); - this.meter = LipSync.createAudioMeter(this.audioContext); - this.mediaStreamSource.connect(this.meter); - } - - // testing only - startFromAudioFile(file) { - if (!this.audioContext) this.audioContext = new AudioContext(); - - if (!this.userSpeechAnalyzer) - this.userSpeechAnalyzer = this.audioContext.createAnalyser(); - this.userSpeechAnalyzer.smoothingTimeConstant = 0.5; - this.userSpeechAnalyzer.fftSize = FFT_SIZE; - - if (this.mediaStreamSource) this.mediaStreamSource.stop(); - - this.audioContext.decodeAudioData(file).then((buffer) => { - this.mediaStreamSource = this.audioContext.createBufferSource(); - this.mediaStreamSource.buffer = buffer; - this.meter = LipSync.createAudioMeter(this.audioContext); - this.mediaStreamSource.connect(this.meter); - this.mediaStreamSource.connect(this.audioContext.destination); - this.mediaStreamSource.start(); - - // connect the output of mediaStreamSource to the input of userSpeechAnalyzer - this.mediaStreamSource.connect(this.userSpeechAnalyzer); - }); - } - - startLipSyncFromAudioBuffer(buffer) { - if (!this.audioContext) this.audioContext = new AudioContext(); - if (!this.userSpeechAnalyzer) - this.userSpeechAnalyzer = this.audioContext.createAnalyser(); - this.userSpeechAnalyzer.smoothingTimeConstant = 0.5; - this.userSpeechAnalyzer.fftSize = FFT_SIZE; - - if (this.mediaStreamSource) this.mediaStreamSource.stop(); - - this.buffer = buffer; - } - - destroy() { - this.meter?.shutdown(); - this.meter = null; - this.mediaStreamSource?.disconnect(); - return this.audioContext?.close().catch(() => {}) || Promise.resolve(); - } - - update(frame, offset, rendering) { - const frameBuffer = 12; - - let pos = frame - offset; - - if (Math.floor(pos) <= 1){ - this.last_frame = 0; - } - - let doPlay = true; - if ( - Math.abs(this.last_frame - frame) < frameBuffer - 1 && - this.last_frame !== 0 - ) { - doPlay = false; - } - - if (doPlay) { - const startTime = pos / 60.0; - const endTime = frameBuffer / 60.0; - this.mediaStreamSource = this.audioContext.createBufferSource(); - this.mediaStreamSource.buffer = this.buffer; - this.meter = LipSync.createAudioMeter(this.audioContext); - this.mediaStreamSource.connect(this.meter); - this.mediaStreamSource.connect(this.audioContext.destination); - this.mediaStreamSource.start(0, startTime, endTime); - this.mediaStreamSource.connect(this.userSpeechAnalyzer); - this.last_frame = frame; - } - - if (this.meter) { - const { volume } = this.meter; - let { ah, oh, ee } = this.update2(); - if (volume < 0.01) { - ah = 0; - oh = 0; - ee = 0; - } - return { ah: ah, oh: oh, ee: ee }; - } - return { ah: 0, oh: 0, ee: 0 }; - } - - update2() { - function getSensitivityMap(spectrum) { - const sensitivity_threshold = 0.5; - const stPSD = new Float32Array(spectrum.length); - for (let i = 0; i < spectrum.length; i++) { - stPSD[i] = sensitivity_threshold + (spectrum[i] + 20) / 140; - } - return stPSD; - } - - const spectrum = new Float32Array( - this.userSpeechAnalyzer.frequencyBinCount, - ); - // Populate frequency data for computing frequency intensities - this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time - // Populate time domain for calculating RMS - // userSpeechAnalyzer.getFloatTimeDomainData(spectrum); - // RMS (root mean square) is a better approximation of current input level than peak (just sampling this frame) - // spectrumRMS = getRMS(spectrum); - - const sensitivityPerPole = getSensitivityMap(spectrum); - - // Lower and higher voices have different frequency domains, so we'll separate and max them - const EnergyBinMasc = new Float32Array(BoundingFrequencyMasc.length); - const EnergyBinFem = new Float32Array(BoundingFrequencyFem.length); - - // Masc energy bins (groups of frequency-depending energy) - for (let m = 0; m < BoundingFrequencyMasc.length - 1; m++) { - for ( - let j = IndicesFrequencyMale[m]; - j <= IndicesFrequencyMale[m + 1]; - j++ - ) - if (sensitivityPerPole[j] > 0) - EnergyBinMasc[m] += sensitivityPerPole[j]; - EnergyBinMasc[m] /= IndicesFrequencyMale[m + 1] - IndicesFrequencyMale[m]; - } - - // Fem energy bin - for (let m = 0; m < BoundingFrequencyFem.length - 1; m++) { - for ( - let j = IndicesFrequencyMale[m]; - j <= IndicesFrequencyMale[m + 1]; - j++ - ) - if (sensitivityPerPole[j] > 0) EnergyBinFem[m] += sensitivityPerPole[j]; - EnergyBinMasc[m] /= IndicesFrequencyMale[m + 1] - IndicesFrequencyMale[m]; - EnergyBinFem[m] = - EnergyBinFem[m] / - (IndicesFrequencyFemale[m + 1] - IndicesFrequencyFemale[m]); - } - const oh = - Math.max(EnergyBinFem[1], EnergyBinMasc[1]) > 0.2 - ? 1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2]) - : (1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])) * - 5 * - Math.max(EnergyBinMasc[1], EnergyBinFem[1]); - - const ah = 3 * Math.max(EnergyBinMasc[3], EnergyBinFem[3]); - const ee = - 0.8 * - (Math.max(EnergyBinMasc[1], EnergyBinFem[1]) - - Math.max(EnergyBinMasc[3], EnergyBinFem[3])); - return { oh, ee, ah }; - } - - static createAudioMeter(audioContext) { - const processor = audioContext.createScriptProcessor(512); - processor.onaudioprocess = (event) => { - const buf = event.inputBuffer.getChannelData(0); - const bufLength = buf.length; - let sum = 0; - let x; - - // eslint-disable-next-line no-plusplus - for (let i = 0; i < bufLength; i++) { - x = buf[i]; - if (Math.abs(x) >= processor.clipLevel) { - processor.clipping = true; - processor.lastClip = window.performance.now(); - } - sum += x * x; - } - const rms = Math.sqrt(sum / bufLength); - processor.volume = Math.max(rms, processor.volume * processor.averaging); - }; - processor.clipping = false; - processor.lastClip = 0; - processor.volume = 0; - processor.clipLevel = 0.98; - processor.averaging = 0.95; - processor.clipLag = 750; - - processor.connect(audioContext.destination); - - processor.checkClipping = () => { - if (!processor.clipping) { - return false; - } - if (processor.lastClip + processor.clipLag < window.performance.now()) { - processor.clipping = false; - } - return processor.clipping; - }; - - processor.shutdown = () => { - processor.disconnect(); - processor.onaudioprocess = null; - }; - - return processor; - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/placeholder_manager.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/placeholder_manager.ts deleted file mode 100644 index ac824f5140..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/placeholder_manager.ts +++ /dev/null @@ -1,117 +0,0 @@ -import * as THREE from "three"; -import { TextGeometry } from "three/addons/geometries/TextGeometry.js"; -import { FontLoader } from "three/examples/jsm/loaders/FontLoader.js"; -import { Font } from "three/examples/jsm/loaders/FontLoader.js"; - -export class LoadingPlaceHolder3DModel { - public message_mesh: THREE.Mesh | undefined; - - constructor() { - this.message_mesh = undefined; - } - - public async destroyWithScene(scene: THREE.Scene) { - if (this.message_mesh === undefined) { - return; - } - scene.remove(this.message_mesh); - this.message_mesh.geometry.dispose(); - } - - public async initialize( - message: string, - position: THREE.Vector3, - font: Font, - ): Promise { - return (this.message_mesh = await this.createTextGeometry( - message, - position, - font, - )); - } - - private async createTextGeometry( - text: string, - position: THREE.Vector3, - font: Font, - ): Promise { - if (font) { - const textGeometry = new TextGeometry(text, { - font: font, - size: 0.25, - height: 0.1, - curveSegments: 1, - bevelEnabled: false, - bevelThickness: 1, - bevelSize: 1, - bevelOffset: 1, - bevelSegments: 5, - }); - - const textMaterial = new THREE.MeshPhongMaterial({ color: 0xffffff }); - const message_mesh = new THREE.Mesh(textGeometry, textMaterial); - message_mesh.position.set(position.x, position.y, position.z); - return message_mesh; - } - throw Error(`Could not Create Text Geometry for ${text}`); - } -} - -export class LoadingPlaceHolderManager { - private messages: Map; - private font: Font | undefined; - private scene: THREE.Scene; - - constructor(scene: THREE.Scene) { - this.messages = new Map(); - this.font = undefined; - this.scene = scene; - } - - async initialize() { - this.font = await this.loadFont(); - } - - public async remove(id: string) { - const result = this.messages.get(id); - if (result === undefined) { - return; - } - result.destroyWithScene(this.scene); - this.messages.delete(id); - } - - public async add(id: string, message: string, position: THREE.Vector3) { - const placeholder = new LoadingPlaceHolder3DModel(); - if (this.font === undefined) { - console.log("Forgot to Initialize LoadingPlaceHolderManager"); - return; - } - const placeholder3DObj = await placeholder.initialize( - message, - position, - this.font, - ); - this.scene.add(placeholder3DObj); - this.messages.set(id, placeholder); - } - - // Load the font wrapper - private async loadFont(): Promise { - return new Promise((resolve, reject) => { - const loader = new FontLoader(); - loader.load( - "/resources/fonts/CarterOneRegular.json", - (font) => { - resolve(font); - }, - (progress) => { - console.log(`Font Loaded: ${progress.loaded / progress.total}`); - }, - (error) => { - reject(error); - }, - ); - }); - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/preview_video_generation.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/preview_video_generation.ts deleted file mode 100644 index d16ab9151b..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/preview_video_generation.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager"; - -import { - ProcessStatus, - StreamingProgressResponse, - VideoAudioPreProcessor, -} from "./VideoProcessor/video_audio_preprocessor"; - -class PreviewVideoGeneration { - mediaUploadAPI: MediaUploadApi; - videoAPI: VideoApi; - videoAndAudioPreProcessor: VideoAudioPreProcessor; - - onProgress: (response: StreamingProgressResponse) => void; - - constructor( - onProgress: (response: StreamingProgressResponse) => void, - ) { - this.onProgress = onProgress; - this.videoAndAudioPreProcessor = new VideoAudioPreProcessor(onProgress); - this.mediaUploadAPI = new MediaUploadApi(); - this.videoAPI = new VideoApi(); - } - - // Cancel any part of the processing - public async cancel() {} - - private async stageOneProcessing() {} - - private async stageTwoProcessing() {} - - private async processVideo() { - // check if video is cached or not if not jump to 50% - // collect frames and run preprocessing in the background. - // start polling for percentage change and convert and remap to remainint % - } - - async clearFrameBuffers() {} -} - -export { PreviewVideoGeneration }; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/retargeting.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/retargeting.ts deleted file mode 100644 index 5fc6977c7b..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/retargeting.ts +++ /dev/null @@ -1,97 +0,0 @@ -export class MoveAIResult { - bone: string; - is_special: boolean; - x: number; - y: number; - z: number; - flip: boolean; - only_y: boolean; - - constructor(bone: string, is_special: boolean = false, x: number = 0, y: number = 0, z: number = 0, - flip:boolean=false, only_y:boolean=false) { - this.bone = bone; - this.is_special = is_special; - this.x = x; - this.y = y; - this.z = z; - this.flip = flip; - this.only_y = only_y; - } -} - -export class Retarget { - move_ai_options: { [Key: string]: MoveAIResult } - constructor() { - this.move_ai_options = { - "_1Head.quaternion": new MoveAIResult("mixamorigHead.quaternion"), - "_1Neck.quaternion": new MoveAIResult("mixamorigNeck.quaternion"), - "_Spine.quaternion": new MoveAIResult("mixamorigSpine.quaternion"), - "_1Spine.quaternion": new MoveAIResult("mixamorigSpine1.quaternion"), - "_3Spine.quaternion": new MoveAIResult("mixamorigSpine2.quaternion"), - - "_1Hips.quaternion": new MoveAIResult("mixamorigHips.quaternion", true, -90, 0, 0, false, true), - "_1Hips.position": new MoveAIResult("mixamorigHips.position", true, 1, 0.05, 1), - - "_1LeftFoot.quaternion": new MoveAIResult("mixamorigLeftFoot.quaternion", true, 0, 0, 0, true), - "_1RightFoot.quaternion": new MoveAIResult("mixamorigRightFoot.quaternion", true, 0, 0, 0, true), - - - "_1RightShoulder.quaternion": new MoveAIResult("mixamorigRightShoulder.quaternion", true, 0, 0, 0, false), - "_1LeftShoulder.quaternion": new MoveAIResult("mixamorigLeftShoulder.quaternion", true, 0, 0, 0, false), - "_1LeftArm.quaternion": new MoveAIResult("mixamorigLeftArm.quaternion", true, 0, 0, 0, false), - "_1RightArm.quaternion": new MoveAIResult("mixamorigRightArm.quaternion", true, 0, 0, 0, false), - "_1LeftForeArm.quaternion": new MoveAIResult("mixamorigLeftForeArm.quaternion"), - "_1RightForeArm.quaternion": new MoveAIResult("mixamorigRightForeArm.quaternion"), - - "_1LeftLeg.quaternion": new MoveAIResult("mixamorigLeftLeg.quaternion", true, 0, 0, 0, true), - "_1RightLeg.quaternion": new MoveAIResult("mixamorigRightLeg.quaternion", true, 0, 0, 0, true), - - "_1LeftUpLeg.quaternion": new MoveAIResult("mixamorigLeftUpLeg.quaternion", true, 0, 180, 0), - "_1RightUpLeg.quaternion": new MoveAIResult("mixamorigRightUpLeg.quaternion", true, 0, 180, 0), - - "_1LeftHand.quaternion": new MoveAIResult("mixamorigLeftHand.quaternion"), - "_1RightHand.quaternion": new MoveAIResult("mixamorigRightHand.quaternion"), - - "_1LeftHandThumb1.quaternion": new MoveAIResult("mixamorigLeftHandThumb1.quaternion"), - "_1LeftHandThumb2.quaternion": new MoveAIResult("mixamorigLeftHandThumb2.quaternion"), - "_1LeftHandThumb3.quaternion": new MoveAIResult("mixamorigLeftHandThumb3.quaternion"), - "_1LeftHandIndex1.quaternion": new MoveAIResult("mixamorigLeftHandIndex1.quaternion"), - "_1LeftHandIndex2.quaternion": new MoveAIResult("mixamorigLeftHandIndex2.quaternion"), - "_1LeftHandIndex3.quaternion": new MoveAIResult("mixamorigLeftHandIndex3.quaternion"), - "_1LeftHandMiddle1.quaternion": new MoveAIResult("mixamorigLeftHandMiddle1.quaternion"), - "_1LeftHandMiddle2.quaternion": new MoveAIResult("mixamorigLeftHandMiddle2.quaternion"), - "_1LeftHandMiddle3.quaternion": new MoveAIResult("mixamorigLeftHandMiddle3.quaternion"), - "_1LeftHandRing1.quaternion": new MoveAIResult("mixamorigLeftHandRing1.quaternion"), - "_1LeftHandRing2.quaternion": new MoveAIResult("mixamorigLeftHandRing2.quaternion"), - "_1LeftHandRing3.quaternion": new MoveAIResult("mixamorigLeftHandRing3.quaternion"), - "_1LeftHandPinky1.quaternion": new MoveAIResult("mixamorigLeftHandPinky1.quaternion"), - "_1LeftHandPinky2.quaternion": new MoveAIResult("mixamorigLeftHandPinky2.quaternion"), - "_1LeftHandPinky3.quaternion": new MoveAIResult("mixamorigLeftHandPinky3.quaternion"), - "_1RightHandThumb1.quaternion": new MoveAIResult("mixamorigRightHandThumb1.quaternion"), - "_1RightHandThumb2.quaternion": new MoveAIResult("mixamorigRightHandThumb2.quaternion"), - "_1RightHandThumb3.quaternion": new MoveAIResult("mixamorigRightHandThumb3.quaternion"), - "_1RightHandIndex1.quaternion": new MoveAIResult("mixamorigRightHandIndex1.quaternion"), - "_1RightHandIndex2.quaternion": new MoveAIResult("mixamorigRightHandIndex2.quaternion"), - "_1RightHandIndex3.quaternion": new MoveAIResult("mixamorigRightHandIndex3.quaternion"), - "_1RightHandMiddle1.quaternion": new MoveAIResult("mixamorigRightHandMiddle1.quaternion"), - "_1RightHandMiddle2.quaternion": new MoveAIResult("mixamorigRightHandMiddle2.quaternion"), - "_1RightHandMiddle3.quaternion": new MoveAIResult("mixamorigRightHandMiddle3.quaternion"), - "_1RightHandRing1.quaternion": new MoveAIResult("mixamorigRightHandRing1.quaternion"), - "_1RightHandRing2.quaternion": new MoveAIResult("mixamorigRightHandRing2.quaternion"), - "_1RightHandRing3.quaternion": new MoveAIResult("mixamorigRightHandRing3.quaternion"), - "_1RightHandPinky1.quaternion": new MoveAIResult("mixamorigRightHandPinky1.quaternion"), - "_1RightHandPinky2.quaternion": new MoveAIResult("mixamorigRightHandPinky2.quaternion"), - "_1RightHandPinky3.quaternion": new MoveAIResult("mixamorigRightHandPinky3.quaternion"), - } - - } - - retarget(boneName: string): MoveAIResult { - for (const [key, value] of Object.entries(this.move_ai_options)) { - if (boneName == key) { - return value; - } - } - return new MoveAIResult(boneName); - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/shaders.js b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/shaders.js deleted file mode 100644 index 5b44e2730c..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/shaders.js +++ /dev/null @@ -1,52 +0,0 @@ -// @see https://discourse.threejs.org/t/production-ready-green-screen-with-three-js/23113/2 - -const VERTEX_SHADER = ` -varying vec2 vUv; - -void main() { - vUv = uv; - gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); -} -`; -const FRAGMENT_SHADER = ` -uniform sampler2D tex; -uniform float texWidth; -uniform float texHeight; - -uniform vec3 keyColor; -uniform float similarity; -uniform float smoothness; -uniform float spill; - -varying vec2 vUv; - -// From https://github.com/libretro/glsl-shaders/blob/master/nnedi3/shaders/rgb-to-yuv.glsl -vec2 RGBtoUV(vec3 rgb) { - return vec2( - rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5, - rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5 - ); -} - -vec4 ProcessChromaKey(vec2 texCoord) { - vec4 rgba = texture2D(tex, texCoord); - float chromaDist = distance(RGBtoUV(texture2D(tex, texCoord).rgb), RGBtoUV(keyColor)); - - float baseMask = chromaDist - similarity; - float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5); - rgba.a = fullMask; - - float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5); - float desat = clamp(rgba.r * 0.2126 + rgba.g * 0.7152 + rgba.b * 0.0722, 0., 1.); - rgba.rgb = mix(vec3(desat, desat, desat), rgba.rgb, spillVal); - - return rgba; -} - -void main(void) { - vec2 texCoord = vUv; - gl_FragColor = ProcessChromaKey(texCoord); -} -`; - -export { VERTEX_SHADER, FRAGMENT_SHADER }; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/timeline.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/timeline.ts deleted file mode 100644 index 99552e4835..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Editor/timeline.ts +++ /dev/null @@ -1,950 +0,0 @@ -import * as THREE from "three"; - -import { ClipUI } from "../clips/clip_ui"; - -import Scene from "./scene.js"; -import AudioEngine from "./Engines/audio_engine"; -import TransformEngine from "./Engines/transform_engine"; -import LipSyncEngine from "./Engines/lip_sync_engine"; - -import Queue, { - UnionedActionTypes, - UnionedDataTypes, -} from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "../Queue/QueueNames"; -import { toEngineActions } from "../Queue/toEngineActions"; -import { fromEngineActions } from "../Queue/fromEngineActions"; -import { ClipGroup, ClipType, AssetType } from "~/enums"; -import { CameraAspectRatio, MediaFileType } from "~/pages/PageEnigma/enums"; -import { Keyframe, MediaItem, UpdateTime } from "~/pages/PageEnigma/models"; -import Editor from "~/pages/PageEnigma/Editor/editor"; -import EmotionEngine from "./Engines/emotion_engine"; -import { IGenerationOptions } from "~/pages/PageEnigma/models/generationOptions"; -import { Vector3 } from "three"; - -import { filmLength, outlinerState } from "../signals"; -import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine"; -import { SplatMesh } from "@sparkjsdev/spark"; - -export class TimeLine { - editorEngine: Editor; - timeline_items: ClipUI[]; - - timeline_limit: number; - absolute_end: number; - scrubber_frame_position: number; - is_playing: boolean; - is_repeating: boolean = true; - - // plays audio - audio_engine: AudioEngine; - // key framing - transform_engine: TransformEngine; - // animation engine - animation_engine: CharacterAnimationEngine; - // lip sync engine - lipSync_engine: LipSyncEngine; - // emotion engine - emotion_engine: EmotionEngine; - - // characters - characters: { [key: string]: ClipGroup }; - - scene: Scene; - camera: THREE.Camera | null; - mouse: THREE.Vector2 | undefined; - - current_time: number; - - camera_name: string; - // ensure that the elements are loaded first. - - private debounce_generate_video: boolean; - - constructor( - editor: Editor, - audio_engine: AudioEngine, - transform_engine: TransformEngine, - lipsync_engine: LipSyncEngine, - animation_engine: CharacterAnimationEngine, - emotion_engine: EmotionEngine, - scene: Scene, - camera: THREE.Camera | null, - mouse: THREE.Vector2 | undefined, - camera_name: string, - ) { - this.editorEngine = editor; - this.timeline_items = []; - this.characters = {}; - this.absolute_end = 60 * 12; - this.timeline_limit = this.getTimelineEnd(); - this.camera = camera; - this.mouse = mouse; - - this.is_playing = false; - this.scrubber_frame_position = 0; // in frames into the tl - - // this will be used to play the audio clips - this.audio_engine = audio_engine; - this.transform_engine = transform_engine; - this.lipSync_engine = lipsync_engine; - this.animation_engine = animation_engine; - this.emotion_engine = emotion_engine; - - this.scene = scene; - this.debounce_generate_video = false; - Queue.subscribe( - QueueNames.TO_ENGINE, - "engine", - this.handleTimelineActions.bind(this), - ); - - this.current_time = 0; - this.camera_name = camera_name; - } - - public async updateUI() { - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.UPDATE_TIME_LINE, - data: this.timeline_items, - }); - } - - public async pushEvent(action: fromEngineActions, data: UnionedDataTypes) { - //this.current_time += 0.75; - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.UPDATE_TIME, - data: data, - }); - } - - public isCharacter(uuid: string): boolean { - this.timeline_items.forEach((clip) => { - if (clip.group == ClipGroup.CHARACTER) - this.characters[clip.object_uuid] = ClipGroup.CHARACTER; - }); - - let result: boolean = false; - for (const key in this.characters) { - if (key === uuid) { - result = true; - return true; - } - } - return result; - } - - public async handleTimelineActions(data: { - action: UnionedActionTypes; - data: UnionedDataTypes; - }) { - const action = data.action; - switch (action) { - case toEngineActions.ADD_KEYFRAME: - await this.addKeyFrame(data.data as Keyframe); - break; - case toEngineActions.UPDATE_KEYFRAME: - await this.updateKeyFrame(data.data as Keyframe); - break; - case toEngineActions.DELETE_KEYFRAME: - await this.deleteKeyFrame(data.data as Keyframe); - break; - case toEngineActions.ADD_CLIP: - await this.addClip(data.data as ClipUI); - break; - case toEngineActions.DELETE_CLIP: - await this.deleteClip(data.data as ClipUI); - break; - case toEngineActions.UPDATE_CLIP: - await this.updateClip(data.data as ClipUI); - break; - case toEngineActions.UPDATE_TIME: - await this.scrub(data.data as UpdateTime); - break; - case toEngineActions.MUTE: - await this.mute(data.data as ClipUI, false); - break; - case toEngineActions.UNMUTE: - await this.mute(data.data as ClipUI, true); - break; - // Create operations - case toEngineActions.ADD_CHARACTER: { - const newObject = await this.addCharacter(data.data as MediaItem); - if (newObject) - this.queueNewObjectMessage( - newObject, - data.data as MediaItem, - AssetType.CHARACTER, - ); - const result = this.editorEngine.sceneManager?.render_outliner( - this.characters, - ); - if (result) outlinerState.items.value = result.items; - break; - } - case toEngineActions.ADD_OBJECT: { - const newObject = await this.addObject(data.data as MediaItem); - if (newObject) - this.queueNewObjectMessage( - newObject, - data.data as MediaItem, - newObject.type as AssetType - ); - const result = this.editorEngine.sceneManager?.render_outliner( - this.characters, - ); - if (result) outlinerState.items.value = result.items; - break; - } - case toEngineActions.ADD_SHAPE: { - const newShape = await this.addShape(data.data as MediaItem); - this.queueNewObjectMessage( - newShape, - data.data as MediaItem, - AssetType.SHAPE, - ); - const result = this.editorEngine.sceneManager?.render_outliner( - this.characters, - ); - if (result) outlinerState.items.value = result.items; - break; - } - case toEngineActions.ENTER_PREVIEW_STATE: - await this.editorEngine.switchPreview(); - break; - case toEngineActions.ENTER_EDIT_STATE: - this.editorEngine.switchEdit(); - break; - case toEngineActions.TOGGLE_CAMERA_STATE: - this.editorEngine.switchCameraView(); - break; - case toEngineActions.TOGGLE_REPEATING: - this.is_repeating = !this.is_repeating; - break; - case toEngineActions.REFRESH_PREVIEW: - if (this.editorEngine.switchPreviewToggle) { - await this.editorEngine.generateFrame(); - } - break; - case toEngineActions.GENERATE_VIDEO: { - console.log("Calling Generate Video"); - // debounce generate video ... - if (this.debounce_generate_video == false) { - this.debounce_generate_video = true; - const options = data.data; // super overloaded talk to the devs about this. TODO... refactor - // pass this in ... rather than doing it implicitly ... - this.editorEngine.generation_options = options as IGenerationOptions; - - await this.editorEngine.generateVideo(); - this.debounce_generate_video = false; - } - console.log("Generate Video Press Event"); - break; - } - case toEngineActions.CHANGE_CAMERA_ASPECT_RATIO: { - this.editorEngine.changeRenderCameraAspectRatio( - data.data as CameraAspectRatio, - ); - break; - } - default: - console.log("Action Not Wired", action); - } - } - - public async addCharacter(data: MediaItem) { - const media_id = data.media_id; - const name = data.name; - const pos = this.getPos(); - const new_data = { ...data }; - - console.debug('Adding to engine scene manager: ', media_id, name, pos); - - const obj = await this.editorEngine.sceneManager?.create( - media_id, - name, - pos, - ); - - if (obj) { - obj.userData["name"] = name; - obj.name = name; - obj.position.copy(pos); - obj.userData.isCharacter = true; - - console.debug("Added character object to timeline; object: ", obj); - - // TODO(bt,2025-01-28): Hack for debugging only. Remove soon. - (window as any).lastCharacter = obj; - - // TODO(bt,2025-01-28): Hack for debugging only. Remove soon. - const lastRig = obj.children[0]; - if (!!lastRig && lastRig.name.includes("gamerig")) { - (window as any).lastRig = lastRig; - } - - const object_uuid = obj.uuid; - - this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character. - new_data["object_uuid"] = object_uuid; - - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.UPDATE_CHARACTER_ID, - data: new_data, - }); - - this.addPlayableClip( - new ClipUI( - data.version, - ClipType.FAKE, - ClipGroup.CHARACTER, - "Default", - media_id, - obj.uuid, - obj.uuid, - name, - 0, - 0, - 0, - obj.userData["media_file_type"], - ), - ); - - await this.editorEngine.sceneManager?.add_creation_undostack(obj); - } - return obj; - } - - queueNewObjectMessage( - item: THREE.Object3D, - data: MediaItem, - asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE | AssetType.SPLAT, - ) { - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.ADD_OBJECT, - data: { - media_id: data.media_id, - type: asset_type, - name: item.name, - object_uuid: item.uuid, - version: 1, - } as MediaItem, - }); - - // this.addPlayableClip( - // new ClipUI( - // data["version"], - // ClipType.FAKE, - // ClipGroup.OBJECT, - // "Default", - // data.media_id, - // item.uuid, - // item.uuid, - // item.name, - // 0, - // 0, - // 0, - // this.scene.get_object_by_uuid(item.uuid)?.userData[ - // "media_file_type" - // ], - // ), - // ); - } - - public getPos() { - this.editorEngine.utils.removeTransformControls(true); - const raycaster = new THREE.Raycaster(); - raycaster.layers.enable(0); - raycaster.layers.enable(1); - if (this.editorEngine.mouse && this.camera) { - raycaster.setFromCamera(this.editorEngine.mouse, this.camera); - const intersects = raycaster.intersectObjects( - this.scene.scene.children, - true, - ); - if (intersects.length > 0) { - return intersects[0].point; - } - } - return new THREE.Vector3(0, 0, 0); - } - - public async addObject(data: MediaItem & { position?: { x: number; y: number; z: number } }) { - const dataPos = data.position; - const pos = dataPos - ? new THREE.Vector3(dataPos.x, dataPos.y, dataPos.z) - : this.getPos(); - const media_id = data.media_id; - const name = data.name; - - const obj = await this.editorEngine.sceneManager?.create( - media_id, - name, - pos, - ); - if (obj) { - obj.userData["name"] = name; - obj.name = name; - obj.position.copy(pos); - - await this.editorEngine.sceneManager?.add_creation_undostack(obj); - } - return obj; - } - - // This method is for local dev testing of objects - // and bypasses the queue system, that's kinda redundant anyway - public addLocalSplat(fileBytes: ArrayBuffer | Uint8Array, flipVertical: boolean = false) { - const splatMesh = new SplatMesh({ fileBytes: fileBytes }); - splatMesh.name = `{SplatMesh-${Date.now()}}`; - - if (flipVertical) { - splatMesh.rotateX(Math.PI); - } - - this.editorEngine.sceneManager?.scene.scene.add(splatMesh); - } - - public async addShape(data: MediaItem) { - const pos = this.getPos(); - const parim = await this.editorEngine.create_parim(data.media_id, pos); - await this.editorEngine.sceneManager?.add_creation_undostack(parim); - return parim; - } - - public async addKeyFrame(data: Keyframe) { - // KeyFrame Object - // version: number; - // clip_uuid: string; - // group: ClipGroup; - // object_uuid?: string; - // offset: number; - // position: XYZ - // rotation: XYZ; - // scale: XYZ; - // selected?: boolean; - const data_json = data; - const uuid = data_json.object_uuid; - const keyframe_uuid = data_json.keyframe_uuid; - - let object_name = this.scene.get_object_by_uuid(uuid)?.name; - if (object_name === undefined) { - object_name = "undefined"; - } - - this.transform_engine.addFrame( - uuid, - data_json.offset, - data_json.position as Vector3, - data_json.rotation as Vector3, - data_json.scale as Vector3, - data_json.offset, - data_json.keyframe_uuid, - ); - - await this.addPlayableClip( - new ClipUI( - data_json["version"], - ClipType.TRANSFORM, - data_json["group"], - object_name, - "", - keyframe_uuid, - uuid, - object_name, - 0, - data_json["offset"], - data_json["offset"], - MediaFileType.None, - ), - ); - - const point = this.scene.createPoint( - data_json.position as Vector3, - data_json.rotation as Vector3, - data_json.scale as Vector3, - data_json.keyframe_uuid, - ); - if (this.editorEngine.camera_person_mode) { - point.visible = false; - } - this.checkEditorCanPlay(); - } - - public checkEditorCanPlay() { - this.editorEngine.can_playback = this.getLastClipEnd() > 1; - this.editorEngine.updateSelectedUI(); - } - - // This method is NOT responsible for deletion of the object from scene or other editor parts - // It is only responsible for deleting the object from the timeline and its corresponding elements - public deleteObject(object: THREE.Object3D) { - if (!object) { - return - } - - if (object.name === this.camera_name) { - return; - } - - const object_uuid = object.uuid; - - this.timeline_items.forEach((element) => { - if ( - element.type == ClipType.TRANSFORM && - element.object_uuid == object_uuid - ) { - this.scene.deletePoint(element.clip_uuid); - } - }); - - // TODO: In the future object should have relations that can be deleted inside the engines - // and the engines handle the deletion of objects. - // Character Animation Engine already does this. - this.timeline_items = this.timeline_items.filter( - (element) => element.object_uuid !== object_uuid, - ); - - this.animation_engine.removeCharacter(object); - - // Update react land here. - } - - public async addClip(data: ClipUI) { - const object_uuid = data.object_uuid; - const media_id = data.media_id; - const name = data.name; - const group = data.group; - const version = 1; - const type = data.type; - const offset = data.offset; - const end_offset = data.length + offset; - const object = this.scene.get_object_by_uuid(object_uuid); - const object_name = - object?.name ?? "undefined"; - const clip_uuid = data.clip_uuid; - - switch (type) { - case "animation": - await this.animation_engine.addCharacterAnimationMedia(object!, media_id, data); - this.animation_engine.evaluate(this.current_time, this.timeline_limit); - break; - case "transform": - this.transform_engine.loadObject(object_uuid, data.length); - break; - case "expression": - this.emotion_engine.loadClip(object_uuid, media_id); - break; - case "audio": - if (group === "character") { - this.lipSync_engine.load_object(object_uuid, media_id); - // media id for this as well it can be downloaded - this.addPlayableClip( - new ClipUI( - version, - ClipType.AUDIO, - ClipGroup.CHARACTER, - name, - media_id, - clip_uuid, - object_uuid, - object_name, - offset, - end_offset, - 0, // length - this.scene.get_object_by_uuid(object_uuid)?.userData[ - "media_file_type" - ], - ), - ); - return; - } else { - this.audio_engine.loadClip(media_id); - } - break; - } - - // media id for this as well it can be downloaded - this.addPlayableClip( - new ClipUI( - version, - type, - group, - name, - media_id, - clip_uuid, - object_uuid, - object_name, - offset, - end_offset, // length - 0, - this.scene.get_object_by_uuid(object_uuid)?.userData["media_file_type"], - ), - ); - - this.checkEditorCanPlay(); - } - - public async addSelfAnimationClip( - data: ClipUI, - animation_clip: THREE.AnimationClip, - ) { - const object_uuid = data.object_uuid; - const media_id = data.media_id; - const name = data.name; - const group = data.group; - const version = 1; - const type = data.type; - const offset = data.offset; - const end_offset = data.length + offset; - const object = this.scene.get_object_by_uuid(object_uuid); - const object_name = - object?.name ?? "undefined"; - const clip_uuid = data.clip_uuid; - - this.animation_engine.addCharacterAnimation(object!, animation_clip, data); - - // media id for this as well it can be downloaded - this.addPlayableClip( - new ClipUI( - version, - type, - group, - name, - media_id, - clip_uuid, - object_uuid, - object_name, - offset, - end_offset, // length - 0, - this.scene.get_object_by_uuid(object_uuid)?.userData["media_file_type"], - ), - ); - - this.checkEditorCanPlay(); - this.updateUI(); - } - - public async deleteKeyFrame(data: Keyframe) { - const keyframe_uuid = data.keyframe_uuid; - const object_uuid = data.object_uuid; - this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid); - this.scene.deletePoint(keyframe_uuid); - for (const element of this.timeline_items) { - if ( - element.clip_uuid === keyframe_uuid && - element.object_uuid === object_uuid - ) { - this.timeline_items = this.timeline_items.filter( - (element) => - !( - element.clip_uuid === keyframe_uuid && - element.object_uuid === object_uuid - ), - ); - break; - } - } - this.checkEditorCanPlay(); - } - - public async updateKeyFrame(data: Keyframe) { - const keyframe_uuid = data.keyframe_uuid; - const keyframe_offset = data.offset; - const object_uuid = data.object_uuid; - - const keyframe_pos = data.position; - const keyframe_rot = data.rotation; - const keyframe_scl = data.scale; - - this.transform_engine.clips[object_uuid].setOffset( - keyframe_uuid, - keyframe_offset, - ); - this.transform_engine.clips[object_uuid].setTransform( - keyframe_uuid, - keyframe_pos as Vector3, - keyframe_rot as Vector3, - keyframe_scl as Vector3, - ); - this.scene.updatePoint( - keyframe_uuid, - keyframe_pos as Vector3, - keyframe_rot as Vector3, - ); - this.checkEditorCanPlay(); - } - - public async updateClip(data: ClipUI) { - // only length and offset changes here. - const object_uuid = data.object_uuid; - const media_id = data.media_id; - const offset = data.offset; - const length = data.length + offset; - const clip_uuid = data.clip_uuid; - - for (const element of this.timeline_items) { - if ( - element.media_id === media_id && - element.object_uuid === object_uuid && - element.clip_uuid == clip_uuid - ) { - element.length = length; - element.offset = offset; - } - } - this.checkEditorCanPlay(); - } - - public async deleteClip(data: ClipUI) { - const object_uuid = data.object_uuid; - const media_id = data.media_id; - const clip_uuid = data.clip_uuid; - - // Remove the clip from the items array - for (let i = 0; i < this.timeline_items.length; i++) { - const element = this.timeline_items[i]; - if ( - element.media_id === media_id && - element.object_uuid === object_uuid && - element.clip_uuid == clip_uuid - ) { - this.timeline_items.splice(i, 1); - break; - } - } - - // Once the clip is removed, we need to make sure the engines aren't tracking it either - // TODO: Ideally the engines should just handle this individually and we check them based on clip type/group - // This way we won't need to iterate over all timeline items - if (data.type === ClipType.ANIMATION) { - const object = this.scene.get_object_by_uuid(object_uuid)!; - this.animation_engine.removeAnimation(object, data) - } - - this.checkEditorCanPlay(); - } - - public async mute(data: ClipUI, isMute: boolean) { - this.timeline_items.forEach((element) => { - if (element.group === data.group) { - element.should_play = isMute; - } - }); - } - - public async addPlayableClip(clip: ClipUI): Promise { - this.timeline_items.push(clip); - } - - public async scrub(data: UpdateTime): Promise { - if (this.is_playing) { - return; - } - const value = Math.floor(data.currentTime); - await this.setScrubberPosition(value); - this.current_time = value; - await this.update(); - - if (this.editorEngine.switchPreviewToggle) { - await this.editorEngine.generateFrame(); - } - } - - // public streaming events into the timeline from - public async setScrubberPosition(offset: number) { - this.scrubber_frame_position = offset; // in ms - } - - // should play from the clip that is closest to the to scrubber - public async play(): Promise { - console.log(`Play - Starting Timeline`); - this.is_playing = true; - } - - public async resetScene() { - for (const element of this.timeline_items) { - if (element.type === ClipType.TRANSFORM) { - const object = this.scene.get_object_by_uuid(element.object_uuid); - if (object && this.transform_engine.clips[element.object_uuid]) { - this.transform_engine.clips[element.object_uuid].reset(object); - } - } else if ( - element.type === ClipType.AUDIO && - element.group !== ClipGroup.CHARACTER - ) { - this.audio_engine.loadClip(element.media_id); - this.audio_engine.stopClip(element.media_id); - } else if (element.type === ClipType.ANIMATION) { - this.animation_engine.stop(); - } else if ( - element.type === ClipType.AUDIO && - element.group === ClipGroup.CHARACTER - ) { - this.lipSync_engine.clips[ - element.object_uuid + element.media_id - ].stop(); - this.lipSync_engine.clips[ - element.object_uuid + element.media_id - ].reset(); - } else if (element.type === ClipType.EXPRESSION) { - const object = this.scene.get_object_by_uuid(element.object_uuid); - if (object) - this.emotion_engine.clips[ - element.object_uuid + element.media_id - ].reset(object); - } - } - } - - public getLastClipEnd(): number { - let longest = 0; - for (const element of this.timeline_items) { - if (longest < element.length) { - longest = element.length; - } - } - return longest; - } - - public getTimelineEnd(): number { - return filmLength.peek() * 1000; - } - - // called by the editor update loop on each frame - public async update( - isRendering = false, - delta_time: number = 0, - ): Promise { - //if (this.is_playing === false) return; // start and stop - this.timeline_limit = this.getTimelineEnd(); - if (this.is_playing) { - this.current_time += delta_time * 1000; - this.pushEvent(fromEngineActions.UPDATE_TIME, { - currentTime: this.current_time, - }); - this.scrubber_frame_position = this.current_time; - } - - if (this.scrubber_frame_position <= 0) { - for (const video_plane of this.scene.video_planes) { - video_plane.currentTime = 0; - } - await this.resetScene(); - } - - for (const video_plane of this.scene.video_planes) { - // Caps to 10fps so that the buffering issue is solved and it plays i am not sure how to fix this. - // TODO: Fix buffering and make 30 fps. - const fixedPoint = 1; - - let pb = parseFloat( - (this.scrubber_frame_position / this.editorEngine.cap_fps).toFixed( - fixedPoint, - ), - ); - pb = parseFloat((pb % video_plane.duration).toFixed(fixedPoint)); - if (video_plane.currentTime !== pb) { - video_plane.playbackRate = 6; - video_plane.currentTime = pb; - } - } - - //this.scrubber_frame_position += 1; - //2. allow stopping. - //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. - - // Since the animation engine is newer we can just call an evaluation on it instead of having the timeline process it - this.animation_engine.evaluate(this.current_time, this.timeline_limit); - - // Iterate over the clips and play them as fit - for (const element of this.timeline_items) { - if ( - element.offset <= this.scrubber_frame_position && - this.scrubber_frame_position <= element.length && - element.should_play - ) { - // run async - // element.play() - // remove the element from the list - const object = this.scene.get_object_by_uuid(element.object_uuid); - if (element.type === ClipType.TRANSFORM) { - if (object && this.transform_engine.clips[element.object_uuid]) { - this.transform_engine.clips[element.object_uuid].step( - object, - element.offset, - this.scrubber_frame_position, - this.scene, - ); - element.length = - this.transform_engine.clips[element.object_uuid].length; - } - } else if ( - element.type === ClipType.AUDIO && - element.group !== ClipGroup.CHARACTER && - this.is_playing && - !isRendering - ) { - // if (this.scrubber_frame_position + 1 >= element.length) { - // this.audio_engine.playClip(element.media_id); - // } else { - // await this.audio_engine.step( - // element.object_uuid + element.media_id, - // this.scrubber_frame_position, element.offset); - // } - this.audio_engine.playClip(element.media_id); - await this.audio_engine.step( - element.media_id, - this.scrubber_frame_position, - element.offset, - ); - } else if ( - element.type === ClipType.AUDIO && - element.group === ClipGroup.CHARACTER && - this.is_playing - ) { - if (object) { - await this.lipSync_engine.clips[ - element.object_uuid + element.media_id - ].play(object); - this.lipSync_engine.clips[ - element.object_uuid + element.media_id - ].step(this.scrubber_frame_position, element.offset); - } - } else if (element.type === ClipType.EXPRESSION) { - if (object) { - await this.emotion_engine.clips[ - object.uuid + element.media_id - ].step(this.scrubber_frame_position - element.offset, object); - } - } - } - } - - if ( - this.scrubber_frame_position >= this.timeline_limit && - this.is_playing - ) { - await this.stop(); - return true; - } - - return false; - } - - private async stop(): Promise { - await this.resetScene(); - this.is_playing = false; - console.log(`Stop - Stopping Timeline`); - this.current_time = 0; - this.update(); - this.pushEvent(fromEngineActions.UPDATE_TIME, { - currentTime: this.current_time, - }); - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEnigmaStore.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEnigmaStore.ts new file mode 100644 index 0000000000..ec4812c2fe --- /dev/null +++ b/frontend/apps/artcraft/app/src/pages/PageEnigma/PageEnigmaStore.ts @@ -0,0 +1,394 @@ +import { create } from "zustand"; +import { CameraAspectRatio, EditorStates } from "./enums"; +import { AssetType, AssetFilterOption, ArtStyle } from "~/enums"; +import { styleList } from "./styleList"; + +export type SceneObjectKind = "object" | "character" | "shape"; + +export interface SceneObject { + id: string; + kind: SceneObjectKind; + name: string; + mediaId?: string; + mediaToken?: string; +} + +export interface CameraEntry { + id: string; + label: string; + focalLength: number; + position: { x: number; y: number; z: number }; + rotation: { x: number; y: number; z: number }; + lookAt: { x: number; y: number; z: number }; +} + +export interface FocalLengthDragging { + isDragging: boolean; + focalLength: number; +} + +export interface SelectedSceneObject { + type: AssetType; + id: string; +} + +export type TransformMode = "move" | "rotate" | "scale"; +export type TransformSpace = "world" | "local"; +export type PoseMode = "select" | "edit"; + +export enum DomLevels { + NONE = 0, + INPUT = 1, + PANEL = 2, + DIALOGUE = 3, +} + +export interface HotkeyStatus { + disabled: boolean; + disabledBy: DomLevels; +} + +export interface EditorLoader { + isShowing: boolean; + message: string | undefined; +} + +const DEFAULT_CAMERAS: CameraEntry[] = [ + { + id: "main", + label: "Main View", + focalLength: 17, + position: { x: -2.5, y: 2.5, z: 2.5 }, + rotation: { x: 0, y: 0, z: 0 }, + lookAt: { x: 0, y: 0, z: 0 }, + }, + { + id: "cam2", + label: "Camera 2", + focalLength: 10, + position: { x: 0, y: 0.6, z: 1.5 }, + rotation: { x: 0, y: 0, z: 0 }, + lookAt: { x: 0, y: 0, z: 0 }, + }, +]; + +interface PageEnigmaState { + // scene contents + objects: SceneObject[]; + characters: SceneObject[]; + shapes: SceneObject[]; + selectedObject: SelectedSceneObject | null; + + // cameras + cameras: CameraEntry[]; + selectedCameraId: string; + cameraAspectRatio: CameraAspectRatio; + focalLengthDragging: FocalLengthDragging; + cameraFilter: AssetFilterOption; + + // editor mode + editorState: EditorStates; + transformMode: TransformMode; + transformSpace: TransformSpace; + selectedMode: string; + poseMode: PoseMode; + showPoseControls: boolean; + gridVisible: boolean; + ignoreKeyDelete: boolean; + hotkeyStatus: HotkeyStatus; + isPromptBoxFocused: boolean; + + // layout / panels + sidePanelWidth: number; + sidePanelVisible: boolean; + dndSidePanelWidth: number; + assetModalVisible: boolean; + assetModalVisibleDuringDrag: boolean; + reopenAfterDrag: boolean; + + // overlays + editorLoader: EditorLoader; + editorLetterBox: boolean; + previewSrc: string; + showErrorDialog: boolean; + errorDialogTitle: string; + errorDialogMessage: string; + + // movies + viewMyMovies: boolean; + generateMovieId: string; + + // generation / stylize + generateProgress: number; + selectedArtStyle: ArtStyle; + upscale: boolean; + faceDetail: boolean; + styleStrength: number; + lipSync: boolean; + cinematic: boolean; + enginePreProcessing: boolean; + globalIPAMediaToken: string | null; + adapterImage: string | null; + positivePrompt: string; + negativePrompt: string; + isUserInputPositive: boolean; + isUserInputNegative: boolean; + showNegativePrompt: boolean; + + // ----- actions ----- + + // scene + addObject: (obj: SceneObject) => void; + addCharacter: (obj: SceneObject) => void; + addShape: (obj: SceneObject) => void; + removeSceneObject: (id: string) => void; + setSelectedObject: (sel: SelectedSceneObject | null) => void; + resetScene: () => void; + + // camera + addCamera: (camera: CameraEntry) => void; + updateCamera: (id: string, updates: Partial) => void; + deleteCamera: (id: string) => void; + setSelectedCameraId: (id: string) => void; + setCameraAspectRatio: (ratio: CameraAspectRatio) => void; + setFocalLengthDragging: (state: FocalLengthDragging) => void; + setCameraFilter: (filter: AssetFilterOption) => void; + + // editor mode + setEditorState: (state: EditorStates) => void; + setTransformMode: (mode: TransformMode) => void; + setTransformSpace: (space: TransformSpace) => void; + setSelectedMode: (mode: string) => void; + setPoseMode: (mode: PoseMode) => void; + setShowPoseControls: (visible: boolean) => void; + setGridVisible: (visible: boolean) => void; + setIgnoreKeyDelete: (ignore: boolean) => void; + disableHotkeyInput: (level: DomLevels) => void; + enableHotkeyInput: (level: DomLevels) => void; + setIsPromptBoxFocused: (focused: boolean) => void; + + // layout + setSidePanelWidth: (px: number) => void; + setSidePanelVisible: (visible: boolean) => void; + setDndSidePanelWidth: (px: number) => void; + setAssetModalVisible: (visible: boolean) => void; + setAssetModalVisibleDuringDrag: (visible: boolean) => void; + setReopenAfterDrag: (reopen: boolean) => void; + + // overlays + showEditorLoader: (message?: string) => void; + hideEditorLoader: () => void; + toggleEditorLetterBox: (next?: boolean) => void; + setPreviewSrc: (src: string) => void; + setErrorDialog: (title: string, message: string) => void; + setShowErrorDialog: (show: boolean) => void; + + // movies + setViewMyMovies: (view: boolean) => void; + setGenerateMovieId: (id: string) => void; + + // generation / stylize + setGenerateProgress: (n: number) => void; + setSelectedArtStyle: (style: ArtStyle) => void; + setUpscale: (v: boolean) => void; + setFaceDetail: (v: boolean) => void; + setStyleStrength: (n: number) => void; + setLipSync: (v: boolean) => void; + setCinematic: (v: boolean) => void; + setEnginePreProcessing: (v: boolean) => void; + setGlobalIPAMediaToken: (t: string | null) => void; + setAdapterImage: (s: string | null) => void; + setPositivePrompt: (p: string) => void; + setNegativePrompt: (p: string) => void; + setIsUserInputPositive: (v: boolean) => void; + setIsUserInputNegative: (v: boolean) => void; + setShowNegativePrompt: (v: boolean) => void; + resetSceneGenerationMetadata: () => void; +} + +export const usePageEnigmaStore = create((set, get) => ({ + // initial state + objects: [], + characters: [], + shapes: [], + selectedObject: null, + + cameras: DEFAULT_CAMERAS, + selectedCameraId: "main", + cameraAspectRatio: CameraAspectRatio.HORIZONTAL_3_2, + focalLengthDragging: { isDragging: false, focalLength: 35 }, + cameraFilter: AssetFilterOption.ALL, + + editorState: EditorStates.EDIT, + transformMode: "move", + transformSpace: "world", + selectedMode: "move", + poseMode: "select", + showPoseControls: false, + gridVisible: true, + ignoreKeyDelete: false, + hotkeyStatus: { disabled: false, disabledBy: DomLevels.NONE }, + isPromptBoxFocused: false, + + sidePanelWidth: 0, + sidePanelVisible: false, + dndSidePanelWidth: -1, + assetModalVisible: false, + assetModalVisibleDuringDrag: true, + reopenAfterDrag: false, + + editorLoader: { isShowing: false, message: "Loading Editor Engine 🦊" }, + editorLetterBox: true, + previewSrc: "", + showErrorDialog: false, + errorDialogTitle: "Error!", + errorDialogMessage: "Something went wrong.", + + viewMyMovies: false, + generateMovieId: "", + + generateProgress: -1, + selectedArtStyle: styleList[0].type, + upscale: false, + faceDetail: true, + styleStrength: 0.8, + lipSync: false, + cinematic: true, + enginePreProcessing: false, + globalIPAMediaToken: null, + adapterImage: null, + positivePrompt: "", + negativePrompt: "", + isUserInputPositive: false, + isUserInputNegative: false, + showNegativePrompt: false, + + // scene actions + addObject: (obj) => + set((s) => ({ objects: [...s.objects, obj] })), + addCharacter: (obj) => + set((s) => ({ characters: [...s.characters, obj] })), + addShape: (obj) => + set((s) => ({ shapes: [...s.shapes, obj] })), + removeSceneObject: (id) => + set((s) => ({ + objects: s.objects.filter((o) => o.id !== id), + characters: s.characters.filter((o) => o.id !== id), + shapes: s.shapes.filter((o) => o.id !== id), + selectedObject: + s.selectedObject?.id === id ? null : s.selectedObject, + })), + setSelectedObject: (sel) => set({ selectedObject: sel }), + resetScene: () => + set({ objects: [], characters: [], shapes: [], selectedObject: null }), + + // camera actions + addCamera: (camera) => set((s) => ({ cameras: [...s.cameras, camera] })), + updateCamera: (id, updates) => + set((s) => ({ + cameras: s.cameras.map((c) => (c.id === id ? { ...c, ...updates } : c)), + })), + deleteCamera: (id) => { + if (id === "main") return; + set((s) => ({ + cameras: s.cameras.filter((c) => c.id !== id), + selectedCameraId: s.selectedCameraId === id ? "main" : s.selectedCameraId, + })); + }, + setSelectedCameraId: (id) => set({ selectedCameraId: id }), + setCameraAspectRatio: (ratio) => set({ cameraAspectRatio: ratio }), + setFocalLengthDragging: (state) => set({ focalLengthDragging: state }), + setCameraFilter: (filter) => set({ cameraFilter: filter }), + + // editor mode actions + setEditorState: (state) => set({ editorState: state }), + setTransformMode: (mode) => set({ transformMode: mode }), + setTransformSpace: (space) => set({ transformSpace: space }), + setSelectedMode: (mode) => set({ selectedMode: mode }), + setPoseMode: (mode) => set({ poseMode: mode }), + setShowPoseControls: (visible) => set({ showPoseControls: visible }), + setGridVisible: (visible) => set({ gridVisible: visible }), + setIgnoreKeyDelete: (ignore) => set({ ignoreKeyDelete: ignore }), + disableHotkeyInput: (level) => { + const status = get().hotkeyStatus; + if (status.disabled) { + if (level > status.disabledBy) { + set({ hotkeyStatus: { ...status, disabledBy: level } }); + } + } else { + set({ hotkeyStatus: { disabled: true, disabledBy: level } }); + } + }, + enableHotkeyInput: (level) => { + const status = get().hotkeyStatus; + if (status.disabled && level >= status.disabledBy) { + set({ hotkeyStatus: { disabled: false, disabledBy: DomLevels.NONE } }); + } + }, + setIsPromptBoxFocused: (focused) => set({ isPromptBoxFocused: focused }), + + // layout actions + setSidePanelWidth: (px) => set({ sidePanelWidth: px }), + setSidePanelVisible: (visible) => set({ sidePanelVisible: visible }), + setDndSidePanelWidth: (px) => set({ dndSidePanelWidth: px }), + setAssetModalVisible: (visible) => set({ assetModalVisible: visible }), + setAssetModalVisibleDuringDrag: (visible) => + set({ assetModalVisibleDuringDrag: visible }), + setReopenAfterDrag: (reopen) => set({ reopenAfterDrag: reopen }), + + // overlays actions + showEditorLoader: (message) => + set({ editorLoader: { isShowing: true, message } }), + hideEditorLoader: () => + set((s) => ({ + editorLoader: { isShowing: false, message: s.editorLoader.message }, + })), + toggleEditorLetterBox: (next) => + set((s) => ({ + editorLetterBox: next !== undefined ? next : !s.editorLetterBox, + })), + setPreviewSrc: (src) => set({ previewSrc: src }), + setErrorDialog: (title, message) => + set({ + errorDialogTitle: title, + errorDialogMessage: message, + showErrorDialog: true, + }), + setShowErrorDialog: (show) => set({ showErrorDialog: show }), + + // movies actions + setViewMyMovies: (view) => set({ viewMyMovies: view }), + setGenerateMovieId: (id) => set({ generateMovieId: id }), + + // generation / stylize actions + setGenerateProgress: (n) => set({ generateProgress: n }), + setSelectedArtStyle: (style) => set({ selectedArtStyle: style }), + setUpscale: (v) => set({ upscale: v }), + setFaceDetail: (v) => set({ faceDetail: v }), + setStyleStrength: (n) => set({ styleStrength: n }), + setLipSync: (v) => set({ lipSync: v }), + setCinematic: (v) => set({ cinematic: v }), + setEnginePreProcessing: (v) => set({ enginePreProcessing: v }), + setGlobalIPAMediaToken: (t) => set({ globalIPAMediaToken: t }), + setAdapterImage: (s) => set({ adapterImage: s }), + setPositivePrompt: (p) => set({ positivePrompt: p }), + setNegativePrompt: (p) => set({ negativePrompt: p }), + setIsUserInputPositive: (v) => set({ isUserInputPositive: v }), + setIsUserInputNegative: (v) => set({ isUserInputNegative: v }), + setShowNegativePrompt: (v) => set({ showNegativePrompt: v }), + resetSceneGenerationMetadata: () => + set({ + positivePrompt: "", + negativePrompt: "", + isUserInputPositive: false, + isUserInputNegative: false, + showNegativePrompt: false, + selectedArtStyle: styleList[0].type, + upscale: false, + faceDetail: false, + styleStrength: 0.8, + lipSync: false, + cinematic: false, + enginePreProcessing: false, + }), +})); diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/Queue.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/Queue.ts deleted file mode 100644 index 6d233e354c..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/Queue.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; - -import { CameraAspectRatio } from "../enums"; -import { - QueueClip, - QueueKeyframe, - UpdateTime, - MediaItem, -} from "~/pages/PageEnigma/models"; -import { toTimelineActions } from "./toTimelineActions"; - -import { ClipUI } from "../clips/clip_ui"; -import { IGenerationOptions } from "../models/generationOptions"; -import { ToastDataType } from "~/components"; -import { QueueNames } from "./QueueNames"; - -export type UnionedActionTypes = - | fromEngineActions - | toEngineActions - | toTimelineActions; - -export type UnionedDataTypes = - | QueueClip - | UpdateTime - | QueueKeyframe - | ClipUI[] - | MediaItem - | ToastDataType - | IGenerationOptions - | CameraAspectRatio - | null; - -export type QueueSubscribeType = { - action: UnionedActionTypes; - data: UnionedDataTypes; -}; - -export type SubscribeHandler = (entry: { - action: UnionedActionTypes; - data: UnionedDataTypes; -}) => void; - -class Queue { - private _queues: Record< - QueueNames, - Array<{ - action: UnionedActionTypes; - data: UnionedDataTypes; - }> - > = { - [QueueNames.FROM_ENGINE]: [], - [QueueNames.TO_ENGINE]: [], - [QueueNames.TO_TIMELINE]: [] - }; - - private _subscribers: Record< - QueueNames, - Map< - string, - SubscribeHandler - > - > = { - [QueueNames.FROM_ENGINE]: new Map(), - [QueueNames.TO_ENGINE]: new Map(), - [QueueNames.TO_TIMELINE]: new Map(), - }; - - // Enqueues actions to a specified queue - public publish({ - queueName, - action, - data, - }: { - queueName: QueueNames; - action: UnionedActionTypes; - data: UnionedDataTypes; - }) { - // Queues for all QueueNames should be initialized above but this is a safeguard for future enum additions - // Make sure the queue exists - if (!this._queues[queueName]) { - this._queues[queueName] = []; - } - - // Queue the task in the right queue - this._queues[queueName].push({ action, data }); - - // TODO: Ideally the publisher shouldn't deal at all with the subscribers - // It should only publish the events to the right queue - // It also doesn't make sense to fire an event on publish because the operations are then dispersed - // leading to potential race conditions - // FIXME: Add some sort of queue loop using internal event and add queue locks using promises to serialize queue publishes - if (this._subscribers[queueName]) { - this._subscribers[queueName].forEach((subcribeHandler) => - subcribeHandler(this._queues[queueName][0]), - ); - this._queues[queueName].shift(); - } - } - - public subscribe( - queueName: QueueNames, - id: string, - onMessage: (entry: QueueSubscribeType) => void, - ) { - - // Queues for all QueueNames should be initialized above but this is a safeguard for future enum additions - // Make sure the queue exists - if (!this._subscribers[queueName]) { - this._subscribers[queueName] = new Map(); - } - - // Set the handler mapped to the ID - this._subscribers[queueName].set(id, onMessage); - - while (this._queues[queueName]?.length) { - this._subscribers[queueName].forEach((subscribeHandler) => - subscribeHandler(this._queues[queueName][0]), - ); - this._queues[queueName].shift(); - } - } -} - -const queue = new Queue(); - -export default queue; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/QueueNames.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/QueueNames.ts deleted file mode 100644 index 813a726847..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/QueueNames.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum QueueNames { - FROM_ENGINE = "FROM_ENGINE", - TO_ENGINE = "TO_ENGINE", - TO_TIMELINE = "TO_TIMELINE", -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/fromEngineActions.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/fromEngineActions.ts deleted file mode 100644 index 093d913bff..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/fromEngineActions.ts +++ /dev/null @@ -1,14 +0,0 @@ -export enum fromEngineActions { - ADD_OBJECT = "add_object", - CAMERA_ASPECT_RATIO_CHANGED = "camera_aspect_ratio_changed", - // data = CameraAspectRatio - DELETE_OBJECT = "delete_object", - DESELECT_OBJECT = "deselect_object", - RESET_TIMELINE = "reset_timeline", - SELECT_OBJECT = "select_object", - UPDATE_CHARACTER_ID = "update_character_id", - UPDATE_TIME = "update_time", - UPDATE_TIME_LINE = "update_timeline", - POP_A_TOAST = "pop_a_toast", - // data={currentTime: number} -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/index.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/index.ts deleted file mode 100644 index 64ee353a42..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import Queue from "./Queue"; -export default Queue; -export * from "./QueueNames"; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toEngineActions.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toEngineActions.ts deleted file mode 100644 index 7a601329f4..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toEngineActions.ts +++ /dev/null @@ -1,47 +0,0 @@ -export enum toEngineActions { - ADD_CLIP = "add_clip", - // data=QueueClip - ADD_KEYFRAME = "add_keyframe", - - ADD_CHARACTER = "add_character", - - ADD_OBJECT = "add_object", - - ADD_SHAPE = "add_shape", - - // data= QueueKeyframe - DELETE_CLIP = "delete_clip", - // data=QueueClip - DELETE_KEYFRAME = "delete_keyframe", - // data=QueueKeyframe - ENTER_EDIT_STATE = "enter_edit", - // data=null - ENTER_PREVIEW_STATE = "enter_preview", - // data=null - GENERATE_VIDEO = "generate_video", - // data=null - MUTE = "mute", - // data={version, type, group, object_uuid} if lipsync - // data={version, type, group} if global_audio - REFRESH_PREVIEW = "refresh_preview", - // data=QueueClip - TOGGLE_CAMERA_STATE = "toggle_camera", - // data=null - TOGGLE_REPEATING = "toggle_repeating", - // data=null - - CHANGE_CAMERA_ASPECT_RATIO = "change_camera_aspect_ratio", - // data=CamperaAspectRatio - - CAMERA_CHANGED = "camera_changed", - - UNMUTE = "unmute", - // data={version, type, group, object_uuid} if lipsync - // data={version, type, group} if global_audio - UPDATE_CLIP = "update_clip", - // data=QueueClip - UPDATE_KEYFRAME = "update_keyframe", - // data=QueueKeyframe - UPDATE_TIME = "update_time", - // data={currentTime: number} -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toInferenceActions.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toInferenceActions.ts deleted file mode 100644 index b3ed965a6b..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toInferenceActions.ts +++ /dev/null @@ -1,3 +0,0 @@ -export enum toInferenceActions { - ADD_TTS_JOB = "add_tts_job" -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toTimelineActions.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toTimelineActions.ts deleted file mode 100644 index 1602a15177..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/Queue/toTimelineActions.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum toTimelineActions { - ADD_KEYFRAME = "add_keyframe", - GENERATE_PROGRESS = "generate_progress", -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/ClipInterfaces/IAnimationClip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/ClipInterfaces/IAnimationClip.ts deleted file mode 100644 index ef939bf369..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/ClipInterfaces/IAnimationClip.ts +++ /dev/null @@ -1,7 +0,0 @@ -export default interface IAnimationClip { - speed: number; - duration: number; - startTime: number; - endTime: number; - evaluate: (time: number) => void; -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/animation_clip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/animation_clip.ts deleted file mode 100644 index bc15cdbc92..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/animation_clip.ts +++ /dev/null @@ -1,348 +0,0 @@ -import * as THREE from "three"; -import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js"; -import { - FBXLoader, - MMDAnimationHelper, - MMDAnimationHelperMixer, -} from "three/examples/jsm/Addons.js"; -import { MMDLoader } from "three/addons/loaders/MMDLoader.js"; - -import { MoveAIResult, Retarget } from "../Editor/retargeting"; -import IAnimationClip from "./ClipInterfaces/IAnimationClip"; -import { GetCdnOrigin } from "~/api/GetCdnOrigin"; -import { StorytellerApiHostStore } from "@storyteller/api"; - -export class AnimationClip implements IAnimationClip { - version: number; - media_id: string; // comes from the server - object_uuid: string; - type: "animation"; - location: "glb" | "remote"; - speed: number; - duration: number; - startTime: number; - length: number; - clip_name: string; - mixer: THREE.AnimationMixer | undefined; - animation_clip: THREE.AnimationClip | undefined; - clip_action: THREE.AnimationAction | undefined; - special_properties: MoveAIResult[]; - retargeted: boolean; - last_frame: number; - - - private isMMD: boolean = false; - private obj: MMDAnimationHelperMixer | undefined; - private helper: MMDAnimationHelper | undefined; - private mmd_anim_clip: THREE.AnimationClip | undefined; - private pose: Object | undefined; - private processingClip: boolean = false; - private mmd_url: string = ""; - - constructor( - version: number, - media_id: string, - location: "glb" | "remote", - object_uuid: string, - speed: number, - length: number, - clip_name: string, - startTime: number, - duration: number, - ) { - this.version = version; - this.media_id = media_id; - this.type = "animation"; - this.object_uuid = object_uuid; - this.location = location; - this.speed = speed; - this.length = length; - this.startTime = startTime; - this.duration = duration; - this.clip_name = clip_name; - this.animation_clip; - this.mixer; - this.clip_action; - this.special_properties = []; - this.retargeted = false; - this.last_frame = 0; - } - - get endTime() { - return this.startTime + this.duration - } - - evaluate: (time: number) => void; - - async get_media_url() { - //This is for prod when we have the proper info on the url. - const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost(); - const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`; - - const response = await fetch(url); - const json = await JSON.parse(await response.text()); - const bucketPath = json["media_file"]["public_bucket_path"]; - - const media_api_base_url = GetCdnOrigin(); - - const media_url = `${media_api_base_url}${bucketPath}`; - return media_url; - } - - _load_animation(): Promise { - // Return the promise chain starting from `this.get_media_url()` - return this.get_media_url().then((url) => { - // Return a new Promise that resolves with the animation clip - return new Promise((resolve) => { - if (url.includes(".glb")) { - const glbLoader = new GLTFLoader(); - - glbLoader.load(url, (gltf) => { - // Assuming the animation is the first one in the animations array - const animationClip = gltf.animations[0]; - resolve(animationClip); - }); - } else if (url.includes(".fbx")) { - const fbxLoader = new FBXLoader(); - fbxLoader.load(url, (fbx) => { - const animationClip = fbx.animations[0]; - this.retargeted = true; - animationClip.tracks.forEach((track) => { - const retarget = new Retarget(); - const retarget_value = retarget.retarget(track.name); - track.name = retarget_value.bone; - console.log(track); - if (retarget_value.is_special) { - this.special_properties.push(retarget_value); - } - }); - resolve(animationClip); - }); - } else if (url.includes(".vmd")) { - console.log("VMD Loader"); - const root = this.mixer?.getRoot(); - this.isMMD = true; - if (root) { - const mmdLoader = new MMDLoader(); - this.mmd_url = url; - mmdLoader.loadAnimation(url, root as THREE.SkinnedMesh, (mmd) => { - mmd.name = this.mmd_url; - resolve(mmd as THREE.AnimationClip); - }); - console.log("Loaded"); - } - console.log("Dont"); - } - }); - }); - } - - _create_mixer(object: THREE.Object3D) { - this.mixer = new THREE.AnimationMixer(object); - return this.mixer; - } - - async _get_clip() { - if (this.animation_clip == null && this.mixer !== null) { - console.log(this.media_id); - if (this.media_id === "SelfClip") { - const rootObject = this.mixer?.getRoot(); - if (rootObject) - this.animation_clip = ( - rootObject as THREE.Object3D - ).animations[0]; - } else { - this.animation_clip = await this._load_animation(); - } - } - return this.animation_clip; - } - - async play(object: THREE.Object3D) { - if (this.mixer == null) { - this.mixer = this._create_mixer(object); - const anim_clip = await this._get_clip(); - if (anim_clip == undefined) { - return; - } - this.clip_action = this.mixer?.clipAction(anim_clip); - if (this.clip_action && this.isMMD === false) { - if (this.clip_action?.isRunning() == false) { - this.clip_action.play(); - } - } - } - } - - update_bones() { - if (this.retargeted === false) { - return; - } - const rootObject = this.mixer?.getRoot(); - if (rootObject) - for ( - let index_ = 0; - index_ < - (rootObject as THREE.Object3D).children.length; - index_++ - ) { - const child_holder = ( - rootObject as THREE.Object3D - ).children[index_]; - if (child_holder.type == "Bone") { - child_holder.traverse( - (bone: THREE.Object3D) => { - for ( - let index__ = 0; - index__ < this.special_properties.length; - index__++ - ) { - const property = this.special_properties[index__]; - if (property.bone == bone.name + ".quaternion") { - const quat_y = THREE.MathUtils.degToRad(property.y); - bone.rotateY(quat_y); - if (property.only_y == false) { - const quat_x = THREE.MathUtils.degToRad(property.x); - bone.rotateX(quat_x); - const quat_z = THREE.MathUtils.degToRad(property.z); - bone.rotateZ(quat_z); - } else { - if (child_holder.parent) { - child_holder.parent.rotation.x = - THREE.MathUtils.degToRad(-90); - } - } - - if (property.flip) { - bone.rotateX(bone.rotation.x * 2 * -1); - } - } else if (property.bone == bone.name + ".position") { - bone.position.set( - bone.position.x * property.x, - bone.position.y * property.y, - bone.position.z * property.z, - ); - } - } - }, - ); - } - } - } - - private async load_vpd_wrapped(): Promise { - return new Promise((resolve) => { - const poseLoader = new MMDLoader(); - poseLoader.loadVPD("/resources/pose/7.vpd", false, (vpd) => { - resolve(vpd); - }); // End of loadVPD call - }); // End of new Promise - } - - async animate(deltatime: number) { - // Sets the frame of the animation mixer. - this.mixer?.setTime(deltatime); - // Regtargeting for Move.AI - this.update_bones(); - - // Next - if (this.isMMD && this.mixer) { - if (this.obj === undefined) { - // Sets the ik helper and object for IK and animation. - this.helper = (this.mixer.getRoot() as THREE.Object3D).parent?.userData[ - "helper" - ]; - } - - this.obj = this.helper?.objects.get( - this.mixer?.getRoot() as THREE.SkinnedMesh, - ); - - const mesh = this.mixer?.getRoot() as THREE.SkinnedMesh; - - if (this.helper) { - this.helper._restoreBones(mesh); // Privateish js function for resetting bones. - this.obj?.mixer?.setTime(deltatime); // Sets the time. - this.helper._saveBones(mesh); // Saves the bones location for restoration later. - - mesh.updateMatrixWorld(true); // Updates mesh. - this.obj?.ikSolver.update(); // Updates IK Solver. - this.obj?.grantSolver.update(); // Updates Grant which moves the bones. - } - - // Changing of animation. - if ( - this.obj && - this.helper && - this.processingClip === false && - this.obj.mixer && - this.mixer._actions[0]._clip.name !== - this.obj.mixer._actions[0]._clip.name && - this.mixer !== undefined - ) { - // On animation change. - this.processingClip = true; - - const helper_mmd = this.helper.objects.get(mesh); - if (helper_mmd?.mixer !== undefined) { - const clips = helper_mmd.mixer._actions; - clips.forEach((clip) => { - this.helper?.objects.delete(clip._clip); - }); - } - - this.helper.remove(mesh); - mesh.pose(); - - const loader = new MMDLoader(); - loader.loadAnimation(this.mmd_url, mesh, (mmd) => { - mmd.name = this.mmd_url; - this.helper?.add(mesh, { - animation: mmd, - physics: false, - }); - this.processingClip = false; - if (helper_mmd?.mixer !== undefined) { - const clips = helper_mmd.mixer._actions; - clips.forEach((clip) => { - this.helper?.objects.delete(clip._clip); - }); - } - this.obj.mixer = this.mixer; - }); - } - } - } - - async step(deltatime: number, isPlaying: boolean, frame: number) { - if (this.mixer == null) { - return; - } - if (this.retargeted) { - if (isPlaying || Math.floor(frame) != this.last_frame) { - if (Math.floor(frame) != 0) { - await this.animate(deltatime); - } - } - } else { - await this.animate(deltatime); - } - this.last_frame = frame; - } - - stop() { - this.mixer?.setTime(0); - } - - toJSON() { - return { - version: this.version, - media_id: this.media_id, - object_uuid: this.object_uuid, - type: this.type, - speed: this.speed, - length: this.length, - clip_name: this.clip_name, - }; - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/audio_clip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/audio_clip.ts deleted file mode 100644 index 0aef20d66c..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/audio_clip.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { GetCdnOrigin } from "~/api/GetCdnOrigin"; -import Ijson from "~/interfaces/Ijson"; -import { StorytellerApiHostStore } from "@storyteller/api"; - -interface AudioDataInterface { - audioContext: AudioContext; - audioBuffer: AudioBuffer; -} - -class AudioData implements AudioDataInterface { - audioContext: AudioContext; - audioBuffer: AudioBuffer; - source: AudioBufferSourceNode | undefined; - - constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) { - this.audioContext = audioContext; - this.audioBuffer = audioBuffer; - } -} - -export class AudioClip implements Ijson { - version: number; - media_id: string; - type: "audio"; - volume: number; - audio_data: AudioData | undefined; - - constructor(version: number, media_id: string, volume: number) { - this.version = version; - this.media_id = media_id; - this.type = "audio"; - this.volume = volume; - this.download_audio().then((data) => { - this.audio_data = data; - }); - } - - async get_media_url() { - //This is for prod when we have the proper info on the url. - const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost(); - const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`; - - const response = await fetch(url); - const json = await JSON.parse(await response.text()); - const bucketPath = json["media_file"]["public_bucket_path"]; - const media_api_base_url = GetCdnOrigin(); - - const media_url = `${media_api_base_url}${bucketPath}`; - return media_url; - } - - async download_audio() { - const url = await this.get_media_url(); - const audioContext = new AudioContext(); - const response = await fetch(url); - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - return new AudioData(audioContext, audioBuffer); - } - - toJSON() { - return { - version: this.version, - media_id: this.media_id, - type: this.type, - volume: this.volume, - }; - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/clip_ui.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/clip_ui.ts deleted file mode 100644 index 12702fd82a..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/clip_ui.ts +++ /dev/null @@ -1,71 +0,0 @@ -// Clip offsets represent the state of the clip on the timeline as well as what type of clip it is. -// it is created from a media id. -import { MediaFileType } from "~/pages/PageEnigma/enums"; -import { ClipGroup, ClipType } from "~/enums"; -import Ijson from "~/interfaces/Ijson"; - -export class ClipUI implements Ijson { - version: number; - type: ClipType; - group: ClipGroup; - name: string; - media_id: string; - object_uuid: string; - object_name: string; - offset: number; // in frames - length: number; // in frames - should_play: boolean; - clip_uuid: string; - keyframe_offset: number; - media_file_type: MediaFileType; - - constructor( - version: number, - type: ClipType, - group: ClipGroup, - name: string, - media_id: string, - clip_uuid: string, - object_uuid: string, - object_name: string, - offset: number, - length: number, - keyframe_offset: number = 0, - media_file_type: MediaFileType, - ) { - this.version = version; - this.group = group; // Only needed for UI - - this.clip_uuid = clip_uuid; - - this.name = name; // UI - this.type = type; // UI and Animation / Audio / Lipsync / : Engine - - this.object_uuid = object_uuid; // Animation / Audio / Lipsync / : Engine - this.object_name = object_name; - this.media_id = media_id; // Animation / Audio / Lipsync / : Engine - this.offset = offset; - this.length = length; - this.should_play = true; - this.keyframe_offset = keyframe_offset; - this.media_file_type = media_file_type; - } - - toJSON() { - return { - version: this.version, - group: this.group, - name: this.name, - type: this.type, - clip_uuid: this.clip_uuid, - object_uuid: this.object_uuid, - object_name: this.object_name, - media_id: this.media_id, - start_offset: this.offset, - ending_offset: this.length, - keyframe_offset: this.keyframe_offset, - media_file_type: this.media_file_type, - }; - } -} - diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/emotion_clip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/emotion_clip.ts deleted file mode 100644 index 363257186b..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/emotion_clip.ts +++ /dev/null @@ -1,143 +0,0 @@ -import * as THREE from "three"; -import { GetCdnOrigin } from "~/api/GetCdnOrigin"; -import Ijson from "~/interfaces/Ijson"; -import { StorytellerApiHostStore } from "@storyteller/api"; - -interface CsvJson { - [key: string]: string[]; -} - -export class EmotionClip implements Ijson { - version: number; - media_id: string; - type: "expression"; - emotion_json: any; - faces: THREE.Mesh[]; - - constructor(version: number, media_id: string) { - this.version = version; - this.media_id = media_id; - this.type = "expression"; - this.faces = []; - this.download_csv().then((data) => { - this.emotion_json = data; - }); - } - - async get_media_url() { - //This is for prod when we have the proper info on the url. - const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost(); - const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`; - - const response = await fetch(url); - const json = await JSON.parse(await response.text()); - const bucketPath = json["media_file"]["public_bucket_path"]; - const media_api_base_url = GetCdnOrigin(); - const media_url = `${media_api_base_url}${bucketPath}`; - return media_url; - } - - async download_json() { - const url = await this.get_media_url(); - const response = await fetch(url); - return await response.json(); - } - - async download_csv(): Promise { - const url = await this.get_media_url(); - const response = await fetch(url); - const csvText = await response.text(); - - // Parsing the CSV text - const rows = csvText.trim().split("\n"); - const header = rows[0].split(","); - const jsonData: CsvJson = {}; - - header.forEach((key) => (jsonData[key] = [])); - - for (let i = 1; i < rows.length; i++) { - const values = rows[i].split(","); - values.forEach((value, index) => { - jsonData[header[index]].push(value); - }); - } - - return jsonData; - } - - async _detect_face(object: THREE.Object3D): Promise { - this.faces = []; - return new Promise((resolve) => { - object.traverse((c: THREE.Object3D) => { - if (c instanceof THREE.Mesh) { - if (c.morphTargetInfluences && c.morphTargetDictionary) { - const blendShapeIndexE = c.morphTargetDictionary["jawOpen"]; - if (blendShapeIndexE !== undefined) { - this.faces.push(c); - resolve(c); - } - } - } - }); - }); - } - - setBlends(shapes: { [key: string]: number }) { - this.faces.forEach((element: THREE.Mesh) => { - if (element.morphTargetInfluences && shapes) { - Object.keys(shapes).forEach((key) => { - let index = element.morphTargetDictionary?.[key]; - if (index === undefined) { - index = - element.morphTargetDictionary?.[ - key.charAt(0).toLowerCase() + key.slice(1) - ]; - } - if ( - typeof index === "number" && - element.morphTargetInfluences !== undefined - ) { - element.morphTargetInfluences[index] = shapes[key]; - } - }); - } - }); - } - - async reset(object: THREE.Object3D) { - await this._detect_face(object); - const keys: { [key: string]: number } = {}; - if (this.emotion_json === undefined || this.emotion_json === null) { - return; - } - Object.keys(this.emotion_json).forEach((key) => { - keys[key] = 0; - }); - this.setBlends(keys); - } - - async step(frame: number, object: THREE.Object3D) { - if (this.faces.length <= 0) { - await this._detect_face(object); - } - if (this.emotion_json === undefined || this.emotion_json === null) { - return; - } - const keys: { [key: string]: number } = {}; - Object.keys(this.emotion_json).forEach((key) => { - if (Math.floor(frame) > this.emotion_json[key].length) { - return; - } - keys[key] = this.emotion_json[key][Math.floor(frame)]; - }); - this.setBlends(keys); - } - - toJSON() { - return { - version: this.version, - media_id: this.media_id, - type: this.type, - }; - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/lipsync_clip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/lipsync_clip.ts deleted file mode 100644 index 0787bb2ed9..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/lipsync_clip.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { LipSync } from "../Editor/lipsync"; -import * as THREE from "three"; -import { GetCdnOrigin } from "~/api/GetCdnOrigin"; -import Ijson from "~/interfaces/Ijson"; -import { StorytellerApiHostStore } from "@storyteller/api"; - -interface AudioDataInterface { - audioContext: AudioContext; - audioBuffer: AudioBuffer; -} - -class AudioData implements AudioDataInterface { - audioContext: AudioContext; - audioBuffer: AudioBuffer; - source: AudioBufferSourceNode | undefined; - - constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) { - this.audioContext = audioContext; - this.audioBuffer = audioBuffer; - } -} - -class BlendShapeHelper { - ah: number; - ee: number; - oh: number; - constructor(ah: number, ee: number, oh: number) { - this.ah = ah; - this.ee = ee; - this.oh = oh; - } -} - -export class LipSyncClip implements Ijson { - version: number; - media_id: string; - type: "lipsync"; - volume: number; - audio_data: AudioData | undefined; - lipsync: LipSync; - blendshape_helper: BlendShapeHelper; - faces: THREE.Mesh[]; - - constructor(version: number, media_id: string, volume: number) { - this.version = version; - this.media_id = media_id; - this.type = "lipsync"; - this.volume = volume; - this.download_audio().then((data) => { - this.audio_data = data; - }); - this.blendshape_helper = new BlendShapeHelper(0, 0, 0); - // we might need 3 of these one for each character ... - this.lipsync = new LipSync(); - this.faces = []; - } - - // lip sync will be generated through TTS - async get_media_url() { - //This is for prod when we have the proper info on the url. - const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost(); - const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`; - - console.log(`API BASE URL? ${apiSchemeAndHost}`); - console.log(`CALLED URL? ${url}`); - - const response = await fetch(url); - const json = await JSON.parse(await response.text()); - const bucketPath = json["media_file"]["public_bucket_path"]; - const media_api_base_url = GetCdnOrigin(); - //const media_base_url = `${media_api_base_url}/vocodes-public`; - //const media_url = `${media_base_url}${bucketPath}`; - const media_url = `${media_api_base_url}${bucketPath}`; - return media_url; - } - - async download_audio() { - const url = await this.get_media_url(); - const audioContext = new AudioContext(); - const response = await fetch(url); - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - return new AudioData(audioContext, audioBuffer); - } - - async _detect_face(object: THREE.Object3D): Promise { - return new Promise((resolve) => { - object.traverse((c: THREE.Object3D) => { - if (c instanceof THREE.Mesh) { - if (c.morphTargetInfluences && c.morphTargetDictionary) { - let blendShapeIndexE = c.morphTargetDictionary["E"]; - let blendShapeIndexO = c.morphTargetDictionary["O"]; - let blendShapeIndexA = c.morphTargetDictionary["aa"]; - - if (blendShapeIndexE === undefined) { - blendShapeIndexE = c.morphTargetDictionary["お"]; // MMD OH - blendShapeIndexO = c.morphTargetDictionary["ω"]; // MMD O - blendShapeIndexA = c.morphTargetDictionary["あ"]; // MMD A - } - - if (blendShapeIndexE != null) { - this.blendshape_helper = new BlendShapeHelper( - blendShapeIndexA, - blendShapeIndexE, - blendShapeIndexO, - ); - this.faces.push(c); - resolve(c); - } - } - } - }); - }); - } - - async play(object: THREE.Object3D) { - if (this.audio_data?.audioBuffer == null) { - await this.download_audio(); - } - if (this.lipsync.face == null) { - this.lipsync = new LipSync(await this._detect_face(object)); - this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer); - } - } - - stop() { - if (this.lipsync == null) { - return; - } - this.lipsync.destroy(); - } - - setBlends(ah: number, ee: number, oh: number) { - this.faces.forEach((element: THREE.Mesh) => { - if (element.morphTargetInfluences) { - element.morphTargetInfluences[this.blendshape_helper.ee] = ee; - element.morphTargetInfluences[this.blendshape_helper.ah] = ah; - element.morphTargetInfluences[this.blendshape_helper.oh] = oh; - } - }); - } - - step( - frame: number, - offset: number, - // rendering: boolean - ) { - if (this.lipsync == null) { - return; - } - const positions = this.lipsync.update(frame, offset, true); - if (positions) - this.setBlends(positions["ee"], positions["ah"], positions["oh"]); - } - - reset() { - if (this.lipsync.face != undefined) { - this.setBlends(0, 0, 0); - } - this.lipsync = new LipSync(); - } - - toJSON() { - return { - version: this.version, - media_id: this.media_id, - type: this.type, - volume: this.volume, - }; - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/transform_clip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/transform_clip.ts deleted file mode 100644 index 3fdfb7e631..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/clips/transform_clip.ts +++ /dev/null @@ -1,238 +0,0 @@ -import * as THREE from "three"; -import Scene from "../Editor/scene"; - -export class TransformFrame { - position: THREE.Vector3; - rotation: THREE.Vector3; - scale: THREE.Vector3; - offset: number; - keyframe_uuid: string; - - constructor( - position: THREE.Vector3, - rotation: THREE.Vector3, - scale: THREE.Vector3, - offset: number, - keyframe_uuid: string, - ) { - this.position = position; - this.rotation = rotation; - this.scale = scale; - this.offset = offset; - this.keyframe_uuid = keyframe_uuid; - } -} - -export class TransformClip { - version: number; - media_id: string; - object_uuid: string; - type: "transform"; - keyframes: TransformFrame[]; - length: number; - - step_frame: number; - looping: boolean; - - offset: number; - - constructor( - version: number, - object_uuid: string, - length: number, - media_id: string = "", - ) { - this.version = version; - this.media_id = media_id; - this.object_uuid = object_uuid; - this.type = "transform"; - - this.length = length; - - this.keyframes = []; - - this.step_frame = 0; - this.offset = 0; - - this.looping = false; - } - - private findNextNumber(location: number) { - // Sort the transform frames based on their offset in ascending order - this.keyframes.sort((a, b) => a.offset - b.offset); - - let currentKeyframe: TransformFrame | undefined; - let nextKeyframe: TransformFrame | undefined; - - this.length = - this.keyframes[this.keyframes.length - 1].offset + this.offset; - - for (const frame of this.keyframes) { - if (frame.offset <= location) { - currentKeyframe = frame; - } else if (frame.offset > location) { - nextKeyframe = frame; - break; - } - } - - return { currentKeyframe, nextKeyframe }; - } - - public setOffset(keyframe_uuid: string, keyframe_offset: number) { - this.keyframes.forEach((item) => { - if (item.keyframe_uuid === keyframe_uuid) { - item.offset = keyframe_offset; - return; - } - }); - } - - public setTransform( - keyframe_uuid: string, - position: THREE.Vector3, - rotation: THREE.Vector3, - scale: THREE.Vector3, - ) { - this.keyframes.forEach((item) => { - if (item.keyframe_uuid === keyframe_uuid) { - item.position = position; - item.rotation = rotation; - item.scale = scale; - return; - } - }); - } - - public removeKeyframe(keyframe_uuid: string) { - this.keyframes = this.keyframes.filter( - (item) => item.keyframe_uuid !== keyframe_uuid, - ); - } - - request_point_location(keyframe_uuid: string, scene: Scene) { - const point = scene.getPoint(keyframe_uuid); - if (point == undefined) { - console.log("Could not find point."); - return; - } - this.keyframes.forEach((keyframe) => { - if (keyframe.keyframe_uuid == keyframe_uuid) { - keyframe.position = point.position; - keyframe.rotation = new THREE.Vector3( - point.rotation.x, - point.rotation.y, - point.rotation.z, - ); - keyframe.scale = point.scale; - } - }); - return point; - } - - step(object: THREE.Object3D, offset: number, frame: number, scene: Scene) { - this.offset = offset; - this.step_frame = frame - offset; - //if (this.step_frame < 0) { this.step_frame = 0; } - //if (this.step_frame >= this.length) { return; } // Reached max frames. - if (this.keyframes.length < 1) { - return; - } // If there are enough points in the scene. - - // Find the current and next keyframes based on time_frame - const { currentKeyframe, nextKeyframe } = this.findNextNumber( - this.step_frame, - ); - if (nextKeyframe != undefined && currentKeyframe != undefined) { - const small_step_frame = this.step_frame - currentKeyframe.offset; - let location = - small_step_frame / (nextKeyframe.offset - currentKeyframe.offset); - if (location < 0) { - location = 0; - } - const pos_s = currentKeyframe?.position; - let pos_e = nextKeyframe?.position; - if (pos_e === undefined) { - this.request_point_location(nextKeyframe.keyframe_uuid, scene); - pos_e = nextKeyframe?.position; - } - const points_pos = [ - new THREE.Vector3(pos_s.x, pos_s.y, pos_s.z), - new THREE.Vector3(pos_e.x, pos_e.y, pos_e.z), - ]; - const curve_pos = new THREE.CatmullRomCurve3(points_pos); - const point_pos = curve_pos.getPoint(location); - object.position.copy(point_pos); - - const rot_s = currentKeyframe?.rotation; - const rot_e = nextKeyframe?.rotation; - const quat_s = new THREE.Quaternion().setFromEuler( - new THREE.Euler( - THREE.MathUtils.degToRad(rot_s.x), - THREE.MathUtils.degToRad(rot_s.y), - THREE.MathUtils.degToRad(rot_s.z), - ), - ); - const quat_e = new THREE.Quaternion().setFromEuler( - new THREE.Euler( - THREE.MathUtils.degToRad(rot_e.x), - THREE.MathUtils.degToRad(rot_e.y), - THREE.MathUtils.degToRad(rot_e.z), - ), - ); - const quaternion = new THREE.Quaternion(); - quaternion.slerpQuaternions(quat_s, quat_e, location); - object.quaternion.copy(quaternion); - - const scale_s = currentKeyframe?.scale; - const scale_e = nextKeyframe?.scale; - const points_scale = [ - new THREE.Vector3(scale_s.x, scale_s.y, scale_s.z), - new THREE.Vector3(scale_e.x, scale_e.y, scale_e.z), - ]; - const curve_scale = new THREE.CatmullRomCurve3(points_scale); - const point_scale = curve_scale.getPoint(location); - object.scale.copy(point_scale); - } - } - - reset(object: THREE.Object3D) { - if (this.keyframes.length > 0) { - const first_pos = this.keyframes[0].position; - const first_rot = this.keyframes[0].rotation; - const first_quat = new THREE.Euler( - THREE.MathUtils.degToRad(first_rot.x), - THREE.MathUtils.degToRad(first_rot.y), - THREE.MathUtils.degToRad(first_rot.z), - ); - const first_scl = this.keyframes[0].scale; - object.position.copy(first_pos); - object.rotation.copy(first_quat); - object.scale.copy(first_scl); - this.step_frame = 0; - } - } - - add_frame( - position: THREE.Vector3, - rotation: THREE.Vector3, - scale: THREE.Vector3, - offset: number, - keyframe_uuid: string, - ) { - this.keyframes.push( - new TransformFrame(position, rotation, scale, offset, keyframe_uuid), - ); - } - - toJSON(): any { - return { - version: this.version, - media_id: this.media_id, - object_uuid: this.object_uuid, - length: this.length, - type: this.type, - keyframes: this.keyframes, - }; - } -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/composable_engines/audio_engine.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/composable_engines/audio_engine.tsx deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/composable_engines/lipsync_engine.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/composable_engines/lipsync_engine.tsx deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx deleted file mode 100644 index 6b5c3af191..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx +++ /dev/null @@ -1,158 +0,0 @@ -import { useContext, useState } from "react"; -import { useSignals } from "@preact/signals-react/runtime"; -import { - faBackwardStep, - faForwardStep, - faCirclePause, - faCirclePlay, - faPlay, - faRepeat, -} from "@fortawesome/pro-solid-svg-icons"; -import { ButtonIcon } from "@storyteller/ui-button-icon"; -import { Tooltip } from "@storyteller/ui-tooltip"; -import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; -import { - currentTime, - filmLength, - secondaryScrubber, -} from "~/pages/PageEnigma/signals"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import { editorState } from "~/pages/PageEnigma/signals/engine"; -import { EditorStates } from "~/pages/PageEnigma/enums"; -import { twMerge } from "tailwind-merge"; -import { usePosthogFeatureFlag } from "~/hooks/usePosthogFeatureFlag"; -import { FeatureFlags } from "~/enums"; - -export const ControlsVideo = () => { - useSignals(); - const editorEngine = useContext(EngineContext); - const [refresh, setRefresh] = useState(0); - const showStylePage = usePosthogFeatureFlag(FeatureFlags.SHOW_STYLE_PAGE); - - function formatTime(millis: number) { - const date = new Date(millis); - const secs = String(date.getUTCSeconds()).padStart(2, "0"); - const ms = String(date.getUTCMilliseconds()).padStart(3, "0"); - return `00:${secs}:${ms}`; - } - - const isPlaying = - editorEngine !== null ? editorEngine.timeline.is_playing : false; - const isRepeating = - editorEngine !== null ? editorEngine.timeline.is_repeating : false; - - const handleToStart = () => { - currentTime.value = 0; - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_TIME, - data: { currentTime: currentTime.value }, - }); - }; - - const handleToEnd = () => { - currentTime.value = filmLength.value * 1000; - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_TIME, - data: { currentTime: currentTime.value }, - }); - }; - - const handleBackwardStep = () => { - currentTime.value = Math.max(currentTime.value - 1, 0); - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_TIME, - data: { currentTime: currentTime.value }, - }); - }; - const handleRepeat = () => { - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.TOGGLE_REPEATING, - data: null, - }); - setRefresh((refresh + 1) % 10); - }; - const handlePlayback = () => { - editorEngine?.togglePlayback(); - }; - const handleForwardStep = () => { - currentTime.value = Math.min( - currentTime.value + 1, - filmLength.value * 1000, - ); - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_TIME, - data: { currentTime: currentTime.value }, - }); - }; - - if (editorState.value === EditorStates.PREVIEW) { - return null; - } - - return ( -
-
-
-
- - -
- - - -
- - -
-
-
-
- - {formatTime(secondaryScrubber.value)} - - / - - {formatTime(filmLength.value * 1000)} - -
- {showStylePage && ( - <> -
- - - - - )} -
-
-
- ); -}; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/index.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/index.tsx deleted file mode 100644 index d4c0c69732..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/ControlsVideo/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export { ControlsVideo } from './ControlsVideo'; \ No newline at end of file diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/TimerGrid/TimerGrid.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/TimerGrid/TimerGrid.tsx deleted file mode 100644 index 2631242bc5..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/comps/TimerGrid/TimerGrid.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import { - filmLength, - frameTrackButtonWidthPx, - fullHeight, - scale, - timelineScrollX, -} from "~/pages/PageEnigma/signals"; -import { Fragment } from "react"; -import { useSignals } from "@preact/signals-react/runtime"; - -export const TimerGrid = () => { - useSignals(); - const sectionWidth = 1000 * 4 * scale.value; - const scrollX = timelineScrollX.value; - - return ( -
-
- {Array(filmLength.value) - .fill(0) - .map((_, index) => ( - -
- 00:{index < 10 ? "0" + index.toString() : index.toString()} -
-
- {Array(9) - .fill(0) - .map((_, ind) => ( -
- ))} - - ))} -
- 00: - {filmLength.value < 10 - ? "0" + filmLength.value.toString() - : filmLength.value.toString()} -
-
-
-
- ); -}; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx deleted file mode 100644 index e9b4bb744d..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import { createContext } from "react"; -import { MediaItem } from "~/pages/PageEnigma/models"; - -export const TrackContext = createContext<{ - // drag and drop - startDrag: (item: MediaItem) => void; - endDrag: () => void; -}>({ - startDrag: () => {}, - endDrag: () => {}, -}); diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx deleted file mode 100644 index d167527ca2..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; -import { ReactNode, useCallback, useMemo } from "react"; -import useUpdateDragDrop from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop"; - -interface Props { - children: ReactNode; -} - -export const TrackProvider = ({ children }: Props) => { - const { endDrag, ...dragDrop } = useUpdateDragDrop(); - - // cross group functions - const dropClip = useCallback(() => { - endDrag(); - }, [endDrag]); - - const values = useMemo(() => { - return { - ...dragDrop, - endDrag: dropClip, - }; - }, [dragDrop, dropClip]); - - return ( - {children} - ); -}; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop.ts deleted file mode 100644 index 2b8673c32e..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { useCallback } from "react"; - -export default function useUpdateDragDrop() { - const startDrag = useCallback(() => {}, []); - - const endDrag = useCallback(() => {}, []); - - return { - startDrag, - endDrag, - }; -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/hooks/useQueueHandler.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/hooks/useQueueHandler.ts deleted file mode 100644 index 5ae99f13c6..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/hooks/useQueueHandler.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; -import { useSignals } from "@preact/signals-react/runtime"; -import { useCallback, useEffect } from "react"; -import { - addNewCharacter, - addObjectToTimeline, - clearExistingData, - currentTime, - deleteObjectOrCharacter, - loadAudioData, - loadCameraData, - loadCharacterData, - loadObjectData, - selectedObject, - addKeyframe, - selectedItem, - generateProgress, -} from "~/pages/PageEnigma/signals"; -import Queue, { QueueSubscribeType } from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; - -import { ToastDataType } from "~/components"; - -import { - MediaItem, - QueueKeyframe, - UpdateTime, -} from "~/pages/PageEnigma/models"; -import { CameraAspectRatio } from "~/pages/PageEnigma/enums"; -import { addToast } from "~/signals"; -import { ToastTypes, ClipGroup } from "~/enums"; -import { toTimelineActions } from "~/pages/PageEnigma/Queue/toTimelineActions"; -import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui"; -import { cameraAspectRatio } from "../signals/engine"; -import { loadPromptTravelData } from "~/pages/PageEnigma/signals/promptTravelGroup"; - -const LOADING_FUNCTIONS: Record void> = { - [ClipGroup.CHARACTER]: loadCharacterData, - [ClipGroup.OBJECT]: loadObjectData, - [ClipGroup.CAMERA]: loadCameraData, - [ClipGroup.GLOBAL_AUDIO]: loadAudioData, - [ClipGroup.PROMPT_TRAVEL]: loadPromptTravelData, -}; - -export function useQueueHandler() { - useSignals(); - - const handleFromEngineActions = useCallback( - ({ action, data }: QueueSubscribeType) => { - //console.log("FROM ENGINE", action, data); - switch (action) { - case fromEngineActions.ADD_OBJECT: { - // this could be an object or character - addObjectToTimeline(data as MediaItem); - break; - } - case fromEngineActions.DELETE_OBJECT: - // this could be an object or character - deleteObjectOrCharacter(data as MediaItem); - break; - case fromEngineActions.DESELECT_OBJECT: - selectedObject.value = null; - break; - case fromEngineActions.RESET_TIMELINE: - clearExistingData(); - break; - case fromEngineActions.SELECT_OBJECT: - selectedObject.value = { - type: (data as MediaItem).type, - id: (data as MediaItem).object_uuid ?? "", - }; - // Deselect clip item when scene item is selected - selectedItem.value = null; - break; - case fromEngineActions.UPDATE_CHARACTER_ID: - addNewCharacter(data as MediaItem); - break; - case fromEngineActions.UPDATE_TIME: - currentTime.value = (data as UpdateTime).currentTime; - break; - case fromEngineActions.UPDATE_TIME_LINE: - clearExistingData(); - (data as ClipUI[]).forEach((item) => { - LOADING_FUNCTIONS[item.group](item); - }); - break; - case fromEngineActions.POP_A_TOAST: { - const message = (data as ToastDataType).message; - addToast(ToastTypes.ERROR, message); - break; - } - case fromEngineActions.CAMERA_ASPECT_RATIO_CHANGED: { - cameraAspectRatio.value = data as CameraAspectRatio; - break; - } - default: - throw new Error(`Unknown action ${action}`); - } - }, - [], - ); - - const handleToTimelineActions = useCallback( - ({ action, data }: QueueSubscribeType) => { - //console.log("TO TIMELINE", action, data); - switch (action) { - case toTimelineActions.ADD_KEYFRAME: - addKeyframe(data as QueueKeyframe, currentTime.value); - break; - case toTimelineActions.GENERATE_PROGRESS: - generateProgress.value = ( - data as { currentTime: number } - ).currentTime; - break; - default: - throw new Error(`Unknown action ${action}`); - } - }, - [], - ); - - useEffect(() => { - Queue.subscribe(QueueNames.FROM_ENGINE, "useQ", handleFromEngineActions); - Queue.subscribe(QueueNames.TO_TIMELINE, "useQ", handleToTimelineActions); - }, [handleFromEngineActions, handleToTimelineActions]); -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/audioGroup.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/audioGroup.ts deleted file mode 100644 index a47ecaf15a..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/audioGroup.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { AudioGroup, Clip, MediaItem } from "~/pages/PageEnigma/models"; -import { ClipGroup, ClipType } from "~/enums"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import * as uuid from "uuid"; -import { signal } from "@preact/signals-core"; -import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui"; -import { publishClip } from "~/pages/PageEnigma/signals/utils/publishClip"; - -export const audioGroup = signal({ - id: "AG-1", - clips: [], - muted: false, -}); - -export function updateAudio({ - id, - offset, - length, -}: { - id: string; - length: number; - offset: number; -}) { - const oldAudioGroup = audioGroup.value; - const newClips = [...oldAudioGroup.clips]; - const clipIndex = newClips.findIndex((row) => row.clip_uuid === id); - if (clipIndex === -1) { - return { ...oldAudioGroup }; - } - const clip = newClips[clipIndex]; - clip.offset = offset; - clip.length = length; - - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_CLIP, - data: clip, - }); - - audioGroup.value = { - ...oldAudioGroup, - clips: newClips, - }; -} - -export function addGlobalAudio({ - dragItem, - audioId, - offset, -}: { - dragItem: MediaItem; - audioId: string; - offset: number; -}) { - if (audioGroup.value.id !== audioId) { - return; - } - - const clip_uuid = uuid.v4(); - const newClip = { - version: 1, - media_id: dragItem.media_id, - type: ClipType.AUDIO, - group: ClipGroup.GLOBAL_AUDIO, - offset, - length: dragItem.length, - name: dragItem.name, - clip_uuid, - } as Clip; - - publishClip(newClip, dragItem, offset); - - const oldAudioGroup = audioGroup.value; - audioGroup.value = { - ...oldAudioGroup, - clips: [...oldAudioGroup.clips, newClip], - }; -} - -export function toggleAudioMute() { - const oldAudioGroup = audioGroup.value; - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: oldAudioGroup?.muted - ? toEngineActions.UNMUTE - : toEngineActions.MUTE, - data: { - version: 1, - type: ClipType.AUDIO, - group: ClipGroup.GLOBAL_AUDIO, - }, - }); - - audioGroup.value = { - ...oldAudioGroup, - muted: !oldAudioGroup.muted, - }; -} - -export function selectAudioClip(clipId: string) { - const oldAudioGroup = audioGroup.value; - audioGroup.value = { - ...oldAudioGroup, - clips: [ - ...oldAudioGroup.clips.map((clip) => { - return { - ...clip, - selected: clip.clip_uuid === clipId ? !clip.selected : clip.selected, - }; - }), - ], - }; -} - -export function deleteAudioClip(clip: Clip) { - const oldAudioGroup = audioGroup.value; - audioGroup.value = { - ...oldAudioGroup, - clips: [ - ...oldAudioGroup.clips.filter((row) => { - if (row.clip_uuid === clip.clip_uuid) { - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.DELETE_CLIP, - data: row, - }); - return false; - } - return true; - }), - ], - }; -} - -export function loadAudioData(item: ClipUI) { - const existingAudio = audioGroup.value; - const newItem = { - version: item.version, - clip_uuid: item.clip_uuid, - type: item.type, - group: item.group, - object_uuid: item.object_uuid, - media_id: item.media_id, - name: item.name, - offset: item.offset, - length: item.length, - } as Clip; - existingAudio.clips.push(newItem); - existingAudio.clips.sort((clipA, clipB) => clipA.offset - clipB.offset); -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/cameraGroup.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/cameraGroup.ts deleted file mode 100644 index 28085c2d24..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/cameraGroup.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { - CameraGroup, - Keyframe, - QueueKeyframe, -} from "~/pages/PageEnigma/models"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import * as uuid from "uuid"; -import { signal } from "@preact/signals-core"; -import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui"; -import { ToastTypes } from "~/enums"; -import { addToast } from "~/signals"; - -export const cameraGroup = signal({ id: "CG1", keyframes: [] }); - -export function updateCamera({ id, offset }: { id: string; offset: number }) { - const oldCameraGroup = cameraGroup.value; - - const existingKeyframe = oldCameraGroup.keyframes.find((row) => { - return row.offset === offset && row.keyframe_uuid !== id; - }); - - if (existingKeyframe) { - addToast( - ToastTypes.WARNING, - "There can only be one keyframe at this offset.", - ); - return; - } - - const newKeyframes = [...oldCameraGroup.keyframes]; - const keyframe = newKeyframes.find((row) => row.keyframe_uuid === id); - if (!keyframe) { - return; - } - keyframe.offset = offset; - - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_KEYFRAME, - data: keyframe, - }); - - cameraGroup.value = { - ...oldCameraGroup, - keyframes: newKeyframes, - }; -} - -export function addCameraKeyframe(keyframe: QueueKeyframe, offset: number) { - const oldCameraGroup = cameraGroup.value; - - const newKeyframe = { - version: keyframe.version, - keyframe_uuid: uuid.v4(), - group: keyframe.group, - object_uuid: keyframe.object_uuid, - offset, - position: keyframe.position, - rotation: keyframe.rotation, - scale: keyframe.scale, - selected: false, - } as Keyframe; - - const existingKeyframe = oldCameraGroup.keyframes.find( - (row) => row.offset === offset, - ); - if (existingKeyframe) { - addToast( - ToastTypes.WARNING, - "There can only be one keyframe at this offset.", - ); - newKeyframe.keyframe_uuid = existingKeyframe.keyframe_uuid; - } - - cameraGroup.value = { - ...oldCameraGroup, - keyframes: [ - ...oldCameraGroup.keyframes.filter( - (keyframe) => - keyframe.keyframe_uuid !== existingKeyframe?.keyframe_uuid, - ), - newKeyframe, - ].sort((keyFrameA, keyframeB) => keyFrameA.offset - keyframeB.offset), - }; - - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: existingKeyframe - ? toEngineActions.UPDATE_KEYFRAME - : toEngineActions.ADD_KEYFRAME, - data: newKeyframe, - }); -} - -export function selectCameraKeyframe(keyframeId: string) { - const oldCameraGroup = cameraGroup.value; - cameraGroup.value = { - ...oldCameraGroup, - keyframes: [ - ...oldCameraGroup.keyframes.map((keyframe) => { - return { - ...keyframe, - selected: - keyframe.keyframe_uuid === keyframeId - ? !keyframe.selected - : keyframe.selected, - }; - }), - ], - }; -} - -export function deleteCameraKeyframe(deleteKeyframe: Keyframe) { - const oldCameraGroup = cameraGroup.value; - cameraGroup.value = { - ...oldCameraGroup, - keyframes: [ - ...oldCameraGroup.keyframes.filter((keyframe) => { - if (keyframe.keyframe_uuid === deleteKeyframe.keyframe_uuid) { - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.DELETE_KEYFRAME, - data: keyframe, - }); - return false; - } - return true; - }), - ], - }; -} - -export function loadCameraData(item: ClipUI) { - const existingCamera = cameraGroup.value; - const newKeyframe = { - version: item.version, - keyframe_uuid: item.clip_uuid, - group: item.group, - object_uuid: item.object_uuid, - offset: item.keyframe_offset, - } as Keyframe; - existingCamera.keyframes.push(newKeyframe); - existingCamera.keyframes.sort( - (keyframeA, keyframeB) => keyframeA.offset - keyframeB.offset, - ); -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/index.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/index.ts index f348ce199e..8ea898092f 100644 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/index.ts +++ b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/index.ts @@ -1,5 +1,3 @@ -export * from "./audioGroup"; -export * from "./cameraGroup"; export * from "./characterGroups"; export * from "./demoAssets"; export * from "./editorOverlays"; @@ -15,7 +13,5 @@ export * from "./shape"; export * from "./sidePanel"; export * from "./sizing"; export * from "./stylizeTab"; -export * from "./timing"; -export * from "./timeline"; export * from "./error"; export * from "./assetModal"; diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/promptTravelGroup.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/promptTravelGroup.ts deleted file mode 100644 index 4fea5c486d..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/promptTravelGroup.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { Clip, MediaItem, PromptTravelGroup } from "~/pages/PageEnigma/models"; -import { ClipGroup, ClipType } from "~/enums"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import * as uuid from "uuid"; -import { signal } from "@preact/signals-core"; -import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui"; -import { publishClip } from "~/pages/PageEnigma/signals/utils/publishClip"; - -export const promptTravelGroup = signal({ - id: "AG-1", - clips: [], -}); - -export function updatePromptTravel({ - id, - offset, - length, - name, -}: { - id: string; - length: number; - offset: number; - name: string; -}) { - const oldPromptTravelGroup = promptTravelGroup.value; - const newClips = [...oldPromptTravelGroup.clips]; - const clipIndex = newClips.findIndex((row) => row.clip_uuid === id); - if (clipIndex === -1) { - return { ...oldPromptTravelGroup }; - } - const clip = newClips[clipIndex]; - clip.offset = offset; - clip.length = length; - clip.name = name; - - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_CLIP, - data: clip, - }); - - promptTravelGroup.value = { - ...oldPromptTravelGroup, - clips: newClips, - }; -} - -export function addPromptTravel({ - text, - length, - offset, -}: { - text: string; - length: number; - offset: number; -}) { - const clip_uuid = uuid.v4(); - const newClip = { - version: 1, - type: ClipType.PROMPT_TRAVEL, - group: ClipGroup.PROMPT_TRAVEL, - offset, - length: length, - name: text, - clip_uuid, - } as Clip; - - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.ADD_CLIP, - data: newClip, - }); - - const oldPromptTravelGroup = promptTravelGroup.value; - promptTravelGroup.value = { - ...oldPromptTravelGroup, - clips: [...oldPromptTravelGroup.clips, newClip], - }; -} - -export function deletePromptTravelClip(clip: Clip) { - const oldPromptTravelGroup = promptTravelGroup.value; - promptTravelGroup.value = { - ...oldPromptTravelGroup, - clips: [ - ...oldPromptTravelGroup.clips.filter((row) => { - if (row.clip_uuid === clip.clip_uuid) { - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.DELETE_CLIP, - data: row, - }); - return false; - } - return true; - }), - ], - }; -} - -export function loadPromptTravelData(item: ClipUI) { - const existingPromptTravelId = promptTravelGroup.value; - const newItem = { - version: item.version, - clip_uuid: item.clip_uuid, - type: item.type, - group: item.group, - object_uuid: item.object_uuid, - media_id: item.media_id, - name: item.name, - offset: item.offset, - length: item.length, - } as Clip; - existingPromptTravelId.clips.push(newItem); - existingPromptTravelId.clips.sort( - (clipA, clipB) => clipA.offset - clipB.offset, - ); -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/sizing.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/sizing.ts index 2d62abdce2..24c994bf2e 100644 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/sizing.ts +++ b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/sizing.ts @@ -1,65 +1,3 @@ -import { signal, computed } from "@preact/signals-core"; -import { characterGroup } from "~/pages/PageEnigma/signals/characterGroups"; -import { objectGroup } from "~/pages/PageEnigma/signals/objectGroup"; -import { pageWidth } from "~/signals"; -import { timelineScrollX } from "~/pages/PageEnigma/signals/timing"; +import { signal } from "@preact/signals-core"; -// timeline -export const scale = signal(0.05); -export const filmLength = signal(7); // seconds -export const timelineHeight = signal(0); - -export const cameraMinimized = signal(false); -export const audioMinimized = signal(false); -export const objectsMinimized = signal(false); - -export const characterHeight = computed(() => { - if (!characterGroup.value.characters.length) { - return 0; - } - return ( - characterGroup.value.characters.reduce((totalHeight, character) => { - return totalHeight + (character.minimized ? 47 : 211); - }, 0) + 8 - ); -}); - -export const objectHeight = computed(() => { - if (!objectGroup.value.objects.length) { - return 0; - } - if (objectsMinimized.value) { - return 55; - } - return 47 + objectGroup.value.objects.length * 47 + 8; -}); - -export const fullHeight = computed(() => { - return ( - characterHeight.value + - objectHeight.value + - (cameraMinimized.value ? 47 : 103) + - (audioMinimized.value ? 47 : 103) + - 24 - ); -}); - -export const minimizeIconPosition = signal(20); - -export const timelinePremiumLockPosition = computed(() => { - return Math.min( - pageWidth.value - 2000 + timelineScrollX.value, - fullWidth.value - 24, - ); -}); - -export const fullWidth = computed(() => { - return filmLength.value * 1000 * 4 * scale.value; -}); - -// side panel export const sidePanelWidth = signal(0); - -export const frameTrackButtonWidthPx = 80; -export const dragHandleWidth = 10; -export const unitPx = 4; // Used for front-end dimension calculation between time and px diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timeline.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timeline.ts deleted file mode 100644 index 9623afc1ce..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timeline.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { QueueKeyframe, Keyframe, MediaItem } from "~/pages/PageEnigma/models"; -import { - addCameraKeyframe, - addCharacterKeyframe, - addObjectKeyframe, - audioGroup, - cameraGroup, - characterGroup, - deleteCameraKeyframe, - deleteCharacter, - deleteCharacterKeyframe, - deleteObjectKeyframe, - objectGroup, -} from "~/pages/PageEnigma/signals"; -import { ClipGroup } from "~/enums"; -import { deleteObject } from "~/pages/PageEnigma/signals/objectGroup/deleteObject"; - -const ADD_KEYFRAME: Record< - ClipGroup, - (keyframe: QueueKeyframe, offset: number) => void -> = { - [ClipGroup.CAMERA]: addCameraKeyframe, - [ClipGroup.CHARACTER]: addCharacterKeyframe, - [ClipGroup.OBJECT]: addObjectKeyframe, - [ClipGroup.GLOBAL_AUDIO]: () => {}, - [ClipGroup.PROMPT_TRAVEL]: () => {}, -}; - -const DELETE_KEYFRAME: Record void> = { - [ClipGroup.CAMERA]: deleteCameraKeyframe, - [ClipGroup.CHARACTER]: deleteCharacterKeyframe, - [ClipGroup.OBJECT]: deleteObjectKeyframe, - [ClipGroup.GLOBAL_AUDIO]: () => {}, - [ClipGroup.PROMPT_TRAVEL]: () => {}, -}; - -export function addKeyframe(keyframe: QueueKeyframe, offset: number) { - ADD_KEYFRAME[keyframe.group](keyframe, offset); -} - -export function deleteKeyframe(keyframe: Keyframe) { - DELETE_KEYFRAME[keyframe.group](keyframe); -} - -export function clearExistingData() { - characterGroup.value = { - id: "ChG1", - characters: [], - }; - cameraGroup.value = { - id: "CG1", - keyframes: [], - }; - audioGroup.value = { - id: "AG-1", - clips: [], - muted: false, - }; - objectGroup.value = { - id: "OG1", - objects: [], - }; -} - -export function deleteObjectOrCharacter(item: MediaItem) { - deleteCharacter(item); - deleteObject(item); -} diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timing.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timing.ts deleted file mode 100644 index 854c37a4f5..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/timing.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { signal } from "@preact/signals-core"; - -export const currentTime = signal(0); -// TODO: scrubber time exists to sync the time the scrubber points -// to with the time display on video controls, we could eliminate this -// and calculate everything properly with currentTime, but that would -// be a developer brain struggle for another day. -export const pointerScrubber = signal(0); -export const secondaryScrubber = signal(0); -export const timelineScrollX = signal(0); -export const timelineScrollY = signal(0); -export const stylizeScrollX = signal(0); diff --git a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/utils/publishClip.ts b/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/utils/publishClip.ts deleted file mode 100644 index a20218112d..0000000000 --- a/frontend/apps/artcraft/app/src/pages/PageEnigma/signals/utils/publishClip.ts +++ /dev/null @@ -1,25 +0,0 @@ -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import { filmLength } from "~/pages/PageEnigma/signals"; -import { MediaItem, Clip } from "~/pages/PageEnigma/models"; - -export function publishClip( - newClip: Clip, - dragItem: MediaItem, - offset: number, -) { - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.ADD_CLIP, - data: newClip, - }); - if (offset + (dragItem.length ?? 0) > filmLength.value * 1000) { - newClip.length = filmLength.value * 1000 - offset; - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.UPDATE_CLIP, - data: newClip, - }); - } -}