mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-04 01:18:34 +00:00
PageEnigma refactor checkpoint: delete timeline subsystem and add Zustand store
Phase 1-3 of the 3D page refactor: - Delete the timeline + event-queue subsystem (Queue/, useQueueHandler, Editor/timeline.ts, Editor/Engines/, clips/, and the timeline-only signal slices: timeline, timing, audioGroup, cameraGroup, promptTravelGroup, utils/publishClip) - Delete dead-code files with no live importers: lipsync.js, retargeting.ts, preview_video_generation.ts, shaders.js, placeholder_manager.ts, both composable_engines/*.tsx stubs, contexts/TrackContext/, comps/TimerGrid, comps/ControlsVideo - Add PageEnigmaStore.ts with the full state shape that will replace preact-signals (scene objects, cameras, transform mode, layout, error overlays, generation/stylize state) - Strip signals/index.ts of deleted slice exports and reduce signals/sizing.ts to just sidePanelWidth Branch does not yet compile -- subsequent commits will strip Queue/timeline references from editor.ts, the remaining signal files, and component consumers, then build out actions/ and event hooks.
This commit is contained in:
Tanish Manku committed
1 parent
c1f4ae21e2
commit
1351a40b3d
45 files changed
+395
-4325
No files matched your search
-308
@@ -1,308 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import { Retarget } from "../retargeting";
|
||||
import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
||||
import { get_media_url } from "~/Classes/ApiHelpers";
|
||||
import { MMDLoader } from "three/examples/jsm/loaders/MMDLoader.js";
|
||||
import { ClipUI } from "../../clips/clip_ui";
|
||||
import Ijson from "~/interfaces/Ijson";
|
||||
|
||||
export const START_FRAME_LABEL = "CharacterStartFrameLabel";
|
||||
|
||||
export class CharacterAnimationEngine implements Ijson {
|
||||
version: number;
|
||||
characterAnimations: Map<THREE.Object3D<THREE.Object3DEventMap>, ClipUI[]> = new Map();
|
||||
characterMixers = new Map<THREE.Object3D<THREE.Object3DEventMap>, THREE.AnimationMixer>();
|
||||
|
||||
constructor(version: number) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
toJSON(): unknown {
|
||||
// Return just an array of character object IDs
|
||||
const characterIds = Array.from(this.characterAnimations.keys()).map((character) => character.uuid);
|
||||
return { characters: characterIds };
|
||||
}
|
||||
|
||||
/**
|
||||
* A single character model is capable of tracking multiple animations, in its own track
|
||||
* These tracks can be blended with weights or additively
|
||||
* The weight change can be done via custom interpolation logic
|
||||
*
|
||||
* @param objectUUID The UUID of the object.
|
||||
* @param clip The 3JS animation clip
|
||||
* @param clipUI The clip info
|
||||
*/
|
||||
addCharacterAnimation(character: THREE.Object3D, clip: THREE.AnimationClip, clipUI: ClipUI) {
|
||||
// If character doesn't exist, create a clip object
|
||||
// If it does exist, add a new animation to the existing character by extracting it
|
||||
// TODO: FIXME: Duplicate prevention?
|
||||
clip.name = clipUI.media_id;
|
||||
character.animations.push(clip);
|
||||
console.log("Adding animation track:")
|
||||
console.log(clip);
|
||||
|
||||
if (!this.characterAnimations.has(character)) {
|
||||
this.characterAnimations.set(character, []);
|
||||
this.#ensureMixerExists(character);
|
||||
}
|
||||
|
||||
// Add the clip info to the character map and sort the clips
|
||||
const characterClips = this.characterAnimations.get(character)!;
|
||||
characterClips.push(clipUI);
|
||||
characterClips.sort((a, b) => a.offset - b.offset);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param objectUUID The UUID of the object.
|
||||
* @param mediaId The media ID of the animation.
|
||||
* @param clipUI The clip info
|
||||
*/
|
||||
async addCharacterAnimationMedia(character: THREE.Object3D, mediaId: string, clipUI: ClipUI) {
|
||||
// If character doesn't exist, create a clip object
|
||||
// If it does exist, add a new animation to the existing character by extracting it
|
||||
// TODO: FIXME: Duplicate prevention?
|
||||
const animationTrack = await this.#load_animation(character, mediaId);
|
||||
this.addCharacterAnimation(character, animationTrack, clipUI);
|
||||
}
|
||||
|
||||
#ensureMixerExists(character: THREE.Object3D) {
|
||||
if (!this.characterMixers.has(character)) {
|
||||
this.characterMixers.set(character, new THREE.AnimationMixer(character));
|
||||
}
|
||||
}
|
||||
|
||||
getMixer(character: THREE.Object3D) {
|
||||
this.#ensureMixerExists(character);
|
||||
return this.characterMixers.get(character)!;
|
||||
}
|
||||
|
||||
#interpolateClips(character: THREE.Object3D, timestamp: number, maxTime: number) {
|
||||
const clips = this.characterAnimations.get(character);
|
||||
|
||||
const mixer = this.getMixer(character);
|
||||
|
||||
// Find the two clips we're interpolating between
|
||||
let prevClip: ClipUI | null = null;
|
||||
let nextClip: ClipUI | null = null;
|
||||
const firstClip = clips ? clips[0] : null;
|
||||
const lastClip = clips ? clips[clips.length - 1] : null;
|
||||
|
||||
// If we're before the first clip, interpolate it in from base pose
|
||||
// TODO: Replace with starting frame
|
||||
if (firstClip && firstClip.offset > timestamp) {
|
||||
prevClip = null;
|
||||
nextClip = firstClip;
|
||||
} else if (lastClip && (lastClip.offset + lastClip.length) < timestamp) { // After last clip, interpolate it out
|
||||
prevClip = lastClip;
|
||||
nextClip = null;
|
||||
} else if (clips) {
|
||||
let prevClipIndex = 0;
|
||||
|
||||
// Loop until the next clip is after the timestamp
|
||||
while (prevClipIndex < clips.length - 1 && clips[prevClipIndex + 1].offset < timestamp) {
|
||||
prevClipIndex++;
|
||||
}
|
||||
|
||||
prevClip = clips[prevClipIndex];
|
||||
nextClip = clips[prevClipIndex + 1];
|
||||
}
|
||||
|
||||
// If the clips exist, fetch their actions. If not, evaluation is either at the start or the end.
|
||||
// If evalution is on either sides, check the start/end frame tracks
|
||||
// Otherwise just mark it null
|
||||
const prevAnimationTrack = prevClip ?
|
||||
this.getCharacterAnimationTrack(character, prevClip.media_id) :
|
||||
this.getCharacterAnimationTrack(character, START_FRAME_LABEL);
|
||||
|
||||
// TODO: Replace with ending frame
|
||||
const nextAnimationTrack = nextClip ? this.getCharacterAnimationTrack(character, nextClip.media_id) : null;
|
||||
|
||||
const prevAction = prevAnimationTrack ? mixer.clipAction(prevAnimationTrack) : null;
|
||||
const nextAction = nextAnimationTrack ? mixer.clipAction(nextAnimationTrack) : null;
|
||||
|
||||
// Calculate the progress of timestamp from end of prev action to start of next action
|
||||
const left = (prevClip?.offset ?? 0) + (prevClip?.length ?? 0);
|
||||
const right = nextClip?.offset ?? maxTime;
|
||||
|
||||
// Simple Linear interpolation for now
|
||||
// TODO: Take an interpolation dependency, or better yet, write a transition engine
|
||||
const progress = (timestamp - left) / (right - left);
|
||||
prevAction?.setEffectiveWeight(1 - progress);
|
||||
nextAction?.setEffectiveWeight(progress);
|
||||
|
||||
// Make sure we hold that last frame for the previous action
|
||||
if (prevAction) {
|
||||
prevAction.clampWhenFinished = true;
|
||||
}
|
||||
|
||||
// The next action should stay at the first frame
|
||||
if (nextAction) {
|
||||
nextAction.paused = true;
|
||||
}
|
||||
|
||||
// Necessary to ensure the actions are active - the default is inactive, mixer won't do anything
|
||||
prevAction?.play();
|
||||
nextAction?.play();
|
||||
|
||||
// The clip time would still be relative to the previous clip
|
||||
const clipTime = timestamp - (prevClip?.offset ?? 0);
|
||||
mixer.setTime(clipTime / 1000);
|
||||
}
|
||||
|
||||
evaluateCharacter(character: THREE.Object3D, timestamp: number, maxTime: number) {
|
||||
const mixer = this.getMixer(character);
|
||||
const clips = this.characterAnimations.get(character);
|
||||
|
||||
// Find the clip we're in right now
|
||||
const currentClip = clips?.find((clip) => {
|
||||
return clip.offset <= timestamp && clip.offset + clip.length >= timestamp;
|
||||
});
|
||||
|
||||
// If timestamp not in any clip, do nothing.
|
||||
// If timestamp in clip, set mixer to the timestamp inside the clip
|
||||
if (!currentClip) {
|
||||
// Let the interpolation function handle this actions
|
||||
this.#interpolateClips(character, timestamp, maxTime)
|
||||
return;
|
||||
}
|
||||
|
||||
const clipTime = timestamp - currentClip.offset;
|
||||
const animationTrack = this.getCharacterAnimationTrack(character, currentClip.media_id);
|
||||
|
||||
// NOTE: This shouldn't really happen unless the UI was desynced at some point from the engine
|
||||
if (!animationTrack) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Play only this action
|
||||
const animationAction = mixer.clipAction(animationTrack);
|
||||
|
||||
// Since it's the only clip in this timestamp, make it full weight and make sure it's not paused (from interpolation or otherwise)
|
||||
animationAction.setEffectiveWeight(1);
|
||||
animationAction.paused = false;
|
||||
|
||||
// Necessary to ensure the actions are active - the default is inactive, mixer won't do anything
|
||||
animationAction.play();
|
||||
|
||||
mixer.setTime(clipTime / 1000);
|
||||
}
|
||||
|
||||
getCharacterAnimationTrack(character: THREE.Object3D, mediaId: string) {
|
||||
return character.animations.find((clip) => clip.name === mediaId);
|
||||
}
|
||||
|
||||
/** Evaluate all character animations at a given timestamp (milliseconds) */
|
||||
evaluate(timestamp: number, maxTime: number) {
|
||||
this.characterMixers.forEach((_, character) => {
|
||||
this.evaluateCharacter(character, timestamp, maxTime);
|
||||
})
|
||||
}
|
||||
|
||||
stopCharacter(character: THREE.Object3D) {
|
||||
const mixer = this.characterMixers.get(character);
|
||||
console.debug("Stopping all actions for character", character);
|
||||
console.debug(mixer);
|
||||
mixer?.stopAllAction();
|
||||
console.debug(mixer);
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.characterMixers.forEach((mixer) => {
|
||||
mixer.stopAllAction();
|
||||
});
|
||||
}
|
||||
|
||||
async #load_animation(character: THREE.Object3D, mediaId: string): Promise<THREE.AnimationClip> {
|
||||
// Get the file URL and extract the (first) animation track from it
|
||||
// TODO: Support for multiple animations in a single file?
|
||||
const url = await get_media_url(mediaId);
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
if (url.includes(".glb")) {
|
||||
const glbLoader = new GLTFLoader();
|
||||
|
||||
glbLoader.load(url, (gltf) => {
|
||||
const animationClip = gltf.animations[0];
|
||||
resolve(animationClip);
|
||||
});
|
||||
} else if (url.includes(".fbx")) {
|
||||
const fbxLoader = new FBXLoader();
|
||||
fbxLoader.load(url, (fbx) => {
|
||||
const animationClip_1 = fbx.animations[0];
|
||||
animationClip_1.tracks.forEach((track) => {
|
||||
const retarget = new Retarget();
|
||||
const retarget_value = retarget.retarget(track.name);
|
||||
track.name = retarget_value.bone;
|
||||
console.log(track);
|
||||
if (retarget_value.is_special) {
|
||||
// TODO: Revisit special properties later
|
||||
// this.special_properties.push(retarget_value);
|
||||
}
|
||||
});
|
||||
resolve(animationClip_1);
|
||||
});
|
||||
} else if (url.includes(".vmd")) {
|
||||
const mmdLoader = new MMDLoader();
|
||||
mmdLoader.loadAnimation(url, character as THREE.SkinnedMesh, (mmd) => {
|
||||
mmd.name = url;
|
||||
resolve(mmd as THREE.AnimationClip);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
removeCharacter(character: THREE.Object3D) {
|
||||
console.debug("Removing character from animation engine", character);
|
||||
this.characterAnimations.delete(character);
|
||||
this.characterMixers.delete(character);
|
||||
}
|
||||
|
||||
removeAnimation(character: THREE.Object3D, clip: ClipUI) {
|
||||
console.debug("Removing animation from character", character, clip);
|
||||
const clips = this.characterAnimations.get(character)!;
|
||||
|
||||
const clipIndex = clips.findIndex((c) => c.clip_uuid === clip.clip_uuid);
|
||||
if (clipIndex < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove the clip from character's tracked clips
|
||||
clips.splice(clipIndex, 1);
|
||||
|
||||
// Stop all character animations to cleanly reset
|
||||
this.stopCharacter(character);
|
||||
}
|
||||
|
||||
clearStartFrame(character: THREE.Object3D, resetTime?: number, maxTime = 1) {
|
||||
// Stop all character animations
|
||||
this.stopCharacter(character);
|
||||
|
||||
// Check the animations and remove any labelled START_FRAME_LABEL
|
||||
character.animations = character.animations.filter((clip) => clip.name !== START_FRAME_LABEL);
|
||||
|
||||
// If resetTime is passed, evaluate the character to that time
|
||||
if (resetTime) {
|
||||
this.evaluateCharacter(character, resetTime, maxTime);
|
||||
}
|
||||
}
|
||||
|
||||
/** resetTime: The time to reset the character to. This should ideally be the current timeline time. */
|
||||
async createStartFrameAnimation(character: THREE.Object3D, poseHelper: any, url: string, resetTime: number = 0, maxTime: number = 1) {
|
||||
//// Clear any existing start frame
|
||||
//// This will also reset the animation to default pose
|
||||
//this.clearStartFrame(character);
|
||||
|
||||
//// We must create the track data now after resetting to default
|
||||
//const poseData = await poseHelper.extractPoseData(url);
|
||||
//const tracks = poseHelper.inflatePoseDataToTracks(character, poseData);
|
||||
|
||||
//// Create the animation track from the rotation values and hip position
|
||||
//// then add it to the character
|
||||
//const startFrameClip = new THREE.AnimationClip(START_FRAME_LABEL, 0, tracks);
|
||||
//character.animations.push(startFrameClip);
|
||||
|
||||
//// Reset the character evaluation to what was passed in
|
||||
//this.evaluateCharacter(character, resetTime, maxTime);
|
||||
}
|
||||
}
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
// A mapping between the model bones names and the Mediapipe output index
|
||||
export const MixamoPoseMap: Record<string, number> = {
|
||||
"mixamorigLeftArm": 11,
|
||||
"mixamorigRightArm": 12,
|
||||
"mixamorigLeftForeArm": 13,
|
||||
"mixamorigRightForeArm": 14,
|
||||
"mixamorigLeftHand": 15,
|
||||
"mixamorigRightHand": 16,
|
||||
"mixamorigLeftUpLeg": 23,
|
||||
"mixamorigRightUpLeg": 24,
|
||||
"mixamorigLeftLeg": 25,
|
||||
"mixamorigRightLeg": 26,
|
||||
"mixamorigLeftFoot": 27,
|
||||
"mixamorigRightFoot": 28,
|
||||
"mixamorigLeftHandThumb2": 21,
|
||||
"mixamorigRightHandThumb2": 22,
|
||||
}
|
||||
|
||||
export const MixamoInterpolationBoneNames = {
|
||||
Spine: "mixamorigSpine",
|
||||
Spine1: "mixamorigSpine1",
|
||||
Spine2: "mixamorigSpine2",
|
||||
Neck: "mixamorigNeck",
|
||||
LeftShoulderBlade: "mixamorigLeftShoulder",
|
||||
DefLeftUpperArm: "DEF-upper_armL001",
|
||||
DefLeftForeArm: "DEF-forearmL001",
|
||||
RightShoulderBlade: "mixamorigRightShoulder",
|
||||
DefRightUpperArm: "DEF-upper_armR001",
|
||||
DefRightForeArm: "DEF-forearmR001",
|
||||
DefLeftThigh: "DEF-thighL001",
|
||||
DefRightThigh: "DEF-thighR001",
|
||||
}
|
||||
|
||||
export const PoseMap = {
|
||||
NOSE: 0,
|
||||
LEFT_SHOULDER: 11,
|
||||
RIGHT_SHOULDER: 12,
|
||||
LEFT_ELBOW: 13,
|
||||
RIGHT_ELBOW: 14,
|
||||
LEFT_WRIST: 15,
|
||||
RIGHT_WRIST: 16,
|
||||
LEFT_HIP: 23,
|
||||
RIGHT_HIP: 24,
|
||||
LEFT_KNEE: 25,
|
||||
RIGHT_KNEE: 26,
|
||||
LEFT_ANKLE: 27,
|
||||
RIGHT_ANKLE: 28,
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import { AnimationClip } from "../../clips/animation_clip";
|
||||
|
||||
export class AnimationEngine {
|
||||
clips: { [key: string]: AnimationClip } = {};
|
||||
version: number;
|
||||
|
||||
constructor(version: number) {
|
||||
this.clips = {};
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
load_object(object_uuid: string, media_id: string, clip_name: string) {
|
||||
console.debug("load_object", object_uuid, media_id, clip_name);
|
||||
this.clips[object_uuid + media_id] = new AnimationClip(
|
||||
this.version,
|
||||
media_id,
|
||||
"glb",
|
||||
object_uuid,
|
||||
1.0,
|
||||
1.0,
|
||||
clip_name,
|
||||
0.0,
|
||||
0.0,
|
||||
);
|
||||
//this.clips[object_uuid + media_id]._load_animation();
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
import { AudioClip } from "../../clips/audio_clip";
|
||||
|
||||
class AudioEngine {
|
||||
clips: { [key: string]: AudioClip } = {};
|
||||
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
|
||||
version: number;
|
||||
playing: string[];
|
||||
last_frame: number;
|
||||
|
||||
constructor() {
|
||||
this.clips = {};
|
||||
this.playing = [];
|
||||
this.audio_sources = {};
|
||||
this.version = 1.0;
|
||||
this.last_frame = 0;
|
||||
}
|
||||
|
||||
// loads clips into the engine to cache
|
||||
loadClip(audio_media_id: string) {
|
||||
if (this.clips[audio_media_id] != null) {
|
||||
return;
|
||||
}
|
||||
this.clips[audio_media_id] = new AudioClip(
|
||||
this.version,
|
||||
audio_media_id,
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
|
||||
// plays from the timeline.
|
||||
playClip(audio_media_id: string) {
|
||||
if (this.clips[audio_media_id] == null) {
|
||||
this.loadClip(audio_media_id);
|
||||
}
|
||||
}
|
||||
|
||||
async step(audio_media_id: string, frame: number, offset: number) {
|
||||
const frameBuffer = 12;
|
||||
|
||||
const pos = frame - offset;
|
||||
|
||||
let doPlay = true;
|
||||
if (
|
||||
Math.abs(this.last_frame - frame) < frameBuffer - 1 &&
|
||||
this.last_frame !== 0
|
||||
) {
|
||||
doPlay = false;
|
||||
}
|
||||
if (pos <= 1) {
|
||||
doPlay = true;
|
||||
}
|
||||
|
||||
const clip = this.clips[audio_media_id];
|
||||
|
||||
if (doPlay && clip.audio_data?.audioContext) {
|
||||
const startTime = pos / 60.0;
|
||||
const endTime = frameBuffer / 60.0;
|
||||
clip.audio_data.source =
|
||||
clip.audio_data.audioContext.createBufferSource();
|
||||
clip.audio_data.source.buffer = clip.audio_data.audioBuffer;
|
||||
clip.audio_data.source.connect(clip.audio_data.audioContext.destination);
|
||||
clip.audio_data.source.start(0, startTime, endTime);
|
||||
this.last_frame = frame;
|
||||
}
|
||||
}
|
||||
|
||||
// stops the clips
|
||||
stopClip(audio_media_id: string) {
|
||||
const clip = this.clips[audio_media_id];
|
||||
if (clip.audio_data?.source) {
|
||||
clip.audio_data.source.stop();
|
||||
clip.audio_data.source.disconnect();
|
||||
} else {
|
||||
console.warn(
|
||||
`AudioManager: AudioClip with id "${audio_media_id}" not found.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default AudioEngine;
|
||||
@@ -1,24 +0,0 @@
|
||||
import { EmotionClip } from "../../clips/emotion_clip";
|
||||
|
||||
class EmotionEngine {
|
||||
clips: { [key: string]: EmotionClip } = {};
|
||||
version: number;
|
||||
|
||||
constructor(version: number) {
|
||||
this.clips = {};
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
// loads clips into the engine to cache
|
||||
loadClip(object_uuid: string, csv_media_id: string) {
|
||||
if (this.clips[object_uuid + csv_media_id] != null) {
|
||||
return;
|
||||
}
|
||||
this.clips[object_uuid + csv_media_id] = new EmotionClip(
|
||||
this.version,
|
||||
csv_media_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export default EmotionEngine;
|
||||
@@ -1,22 +0,0 @@
|
||||
import { LipSyncClip } from "../../clips/lipsync_clip";
|
||||
|
||||
// needs to be apart of the editor window.
|
||||
export default class LipSyncEngine {
|
||||
clips: { [key: string]: LipSyncClip } = {};
|
||||
audio_sources: { [key: string]: AudioBufferSourceNode } = {};
|
||||
version: number;
|
||||
|
||||
constructor() {
|
||||
this.clips = {};
|
||||
this.audio_sources = {};
|
||||
this.version = 1.0;
|
||||
}
|
||||
|
||||
load_object(object_uuid: string, audio_media_id: string) {
|
||||
this.clips[object_uuid + audio_media_id] = new LipSyncClip(
|
||||
this.version,
|
||||
audio_media_id,
|
||||
1.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
import { TransformClip } from "../../clips/transform_clip";
|
||||
import * as THREE from "three";
|
||||
|
||||
class TransformEngine {
|
||||
clips: { [key: string]: TransformClip } = {};
|
||||
version: number;
|
||||
|
||||
constructor(version: number) {
|
||||
this.version = version;
|
||||
this.clips = {};
|
||||
}
|
||||
|
||||
loadObject(object_uuid: string, clip_length: number = 2) {
|
||||
this.clips[object_uuid] = new TransformClip(
|
||||
this.version,
|
||||
object_uuid,
|
||||
clip_length,
|
||||
object_uuid,
|
||||
); // replace the last object_uuid with the media ID when its ready.
|
||||
return this.clips[object_uuid];
|
||||
}
|
||||
|
||||
addFrame(
|
||||
object_uuid: string,
|
||||
clip_length: number = 2,
|
||||
position: THREE.Vector3,
|
||||
rotation: THREE.Vector3,
|
||||
scale: THREE.Vector3,
|
||||
offset: number,
|
||||
keyframe_uuid: string,
|
||||
): boolean {
|
||||
if (this.clips[object_uuid] == null) {
|
||||
this.clips[object_uuid] = new TransformClip(
|
||||
this.version,
|
||||
object_uuid,
|
||||
clip_length,
|
||||
);
|
||||
this.clips[object_uuid].add_frame(
|
||||
position,
|
||||
rotation,
|
||||
scale,
|
||||
offset,
|
||||
keyframe_uuid,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
this.clips[object_uuid].add_frame(
|
||||
position,
|
||||
rotation,
|
||||
scale,
|
||||
offset,
|
||||
keyframe_uuid,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export default TransformEngine;
|
||||
@@ -1,239 +0,0 @@
|
||||
const BoundingFrequencyMasc = [0, 400, 560, 2400, 4800];
|
||||
const BoundingFrequencyFem = [0, 500, 700, 3000, 6000];
|
||||
const IndicesFrequencyFemale = [];
|
||||
const IndicesFrequencyMale = [];
|
||||
const FFT_SIZE = 1024;
|
||||
const samplingFrequency = 44100;
|
||||
|
||||
for (let m = 0; m < BoundingFrequencyMasc.length; m++) {
|
||||
IndicesFrequencyMale[m] = Math.round(
|
||||
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyMasc[m],
|
||||
);
|
||||
}
|
||||
|
||||
for (let m = 0; m < BoundingFrequencyFem.length; m++) {
|
||||
IndicesFrequencyFemale[m] = Math.round(
|
||||
((2 * FFT_SIZE) / samplingFrequency) * BoundingFrequencyFem[m],
|
||||
);
|
||||
}
|
||||
|
||||
export class LipSync {
|
||||
constructor(face) {
|
||||
this.face = face;
|
||||
this.buffer = null;
|
||||
this.last_frame = 0;
|
||||
|
||||
// WE NEED THIS TO be run at RUNTIME
|
||||
// we have to modify this for our
|
||||
// this.ee = this.face.morphTargetDictionary["E"];
|
||||
// this.oh = this.face.morphTargetDictionary["O"];
|
||||
// this.ah = this.face.morphTargetDictionary["aa"];
|
||||
}
|
||||
|
||||
// for streaming sound?
|
||||
start(stream) {
|
||||
this.audioContext = new AudioContext();
|
||||
this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
|
||||
this.meter = LipSync.createAudioMeter(this.audioContext);
|
||||
this.mediaStreamSource.connect(this.meter);
|
||||
}
|
||||
|
||||
// testing only
|
||||
startFromAudioFile(file) {
|
||||
if (!this.audioContext) this.audioContext = new AudioContext();
|
||||
|
||||
if (!this.userSpeechAnalyzer)
|
||||
this.userSpeechAnalyzer = this.audioContext.createAnalyser();
|
||||
this.userSpeechAnalyzer.smoothingTimeConstant = 0.5;
|
||||
this.userSpeechAnalyzer.fftSize = FFT_SIZE;
|
||||
|
||||
if (this.mediaStreamSource) this.mediaStreamSource.stop();
|
||||
|
||||
this.audioContext.decodeAudioData(file).then((buffer) => {
|
||||
this.mediaStreamSource = this.audioContext.createBufferSource();
|
||||
this.mediaStreamSource.buffer = buffer;
|
||||
this.meter = LipSync.createAudioMeter(this.audioContext);
|
||||
this.mediaStreamSource.connect(this.meter);
|
||||
this.mediaStreamSource.connect(this.audioContext.destination);
|
||||
this.mediaStreamSource.start();
|
||||
|
||||
// connect the output of mediaStreamSource to the input of userSpeechAnalyzer
|
||||
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
|
||||
});
|
||||
}
|
||||
|
||||
startLipSyncFromAudioBuffer(buffer) {
|
||||
if (!this.audioContext) this.audioContext = new AudioContext();
|
||||
if (!this.userSpeechAnalyzer)
|
||||
this.userSpeechAnalyzer = this.audioContext.createAnalyser();
|
||||
this.userSpeechAnalyzer.smoothingTimeConstant = 0.5;
|
||||
this.userSpeechAnalyzer.fftSize = FFT_SIZE;
|
||||
|
||||
if (this.mediaStreamSource) this.mediaStreamSource.stop();
|
||||
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.meter?.shutdown();
|
||||
this.meter = null;
|
||||
this.mediaStreamSource?.disconnect();
|
||||
return this.audioContext?.close().catch(() => {}) || Promise.resolve();
|
||||
}
|
||||
|
||||
update(frame, offset, rendering) {
|
||||
const frameBuffer = 12;
|
||||
|
||||
let pos = frame - offset;
|
||||
|
||||
if (Math.floor(pos) <= 1){
|
||||
this.last_frame = 0;
|
||||
}
|
||||
|
||||
let doPlay = true;
|
||||
if (
|
||||
Math.abs(this.last_frame - frame) < frameBuffer - 1 &&
|
||||
this.last_frame !== 0
|
||||
) {
|
||||
doPlay = false;
|
||||
}
|
||||
|
||||
if (doPlay) {
|
||||
const startTime = pos / 60.0;
|
||||
const endTime = frameBuffer / 60.0;
|
||||
this.mediaStreamSource = this.audioContext.createBufferSource();
|
||||
this.mediaStreamSource.buffer = this.buffer;
|
||||
this.meter = LipSync.createAudioMeter(this.audioContext);
|
||||
this.mediaStreamSource.connect(this.meter);
|
||||
this.mediaStreamSource.connect(this.audioContext.destination);
|
||||
this.mediaStreamSource.start(0, startTime, endTime);
|
||||
this.mediaStreamSource.connect(this.userSpeechAnalyzer);
|
||||
this.last_frame = frame;
|
||||
}
|
||||
|
||||
if (this.meter) {
|
||||
const { volume } = this.meter;
|
||||
let { ah, oh, ee } = this.update2();
|
||||
if (volume < 0.01) {
|
||||
ah = 0;
|
||||
oh = 0;
|
||||
ee = 0;
|
||||
}
|
||||
return { ah: ah, oh: oh, ee: ee };
|
||||
}
|
||||
return { ah: 0, oh: 0, ee: 0 };
|
||||
}
|
||||
|
||||
update2() {
|
||||
function getSensitivityMap(spectrum) {
|
||||
const sensitivity_threshold = 0.5;
|
||||
const stPSD = new Float32Array(spectrum.length);
|
||||
for (let i = 0; i < spectrum.length; i++) {
|
||||
stPSD[i] = sensitivity_threshold + (spectrum[i] + 20) / 140;
|
||||
}
|
||||
return stPSD;
|
||||
}
|
||||
|
||||
const spectrum = new Float32Array(
|
||||
this.userSpeechAnalyzer.frequencyBinCount,
|
||||
);
|
||||
// Populate frequency data for computing frequency intensities
|
||||
this.userSpeechAnalyzer.getFloatFrequencyData(spectrum); // getByteTimeDomainData gets volumes over the sample time
|
||||
// Populate time domain for calculating RMS
|
||||
// userSpeechAnalyzer.getFloatTimeDomainData(spectrum);
|
||||
// RMS (root mean square) is a better approximation of current input level than peak (just sampling this frame)
|
||||
// spectrumRMS = getRMS(spectrum);
|
||||
|
||||
const sensitivityPerPole = getSensitivityMap(spectrum);
|
||||
|
||||
// Lower and higher voices have different frequency domains, so we'll separate and max them
|
||||
const EnergyBinMasc = new Float32Array(BoundingFrequencyMasc.length);
|
||||
const EnergyBinFem = new Float32Array(BoundingFrequencyFem.length);
|
||||
|
||||
// Masc energy bins (groups of frequency-depending energy)
|
||||
for (let m = 0; m < BoundingFrequencyMasc.length - 1; m++) {
|
||||
for (
|
||||
let j = IndicesFrequencyMale[m];
|
||||
j <= IndicesFrequencyMale[m + 1];
|
||||
j++
|
||||
)
|
||||
if (sensitivityPerPole[j] > 0)
|
||||
EnergyBinMasc[m] += sensitivityPerPole[j];
|
||||
EnergyBinMasc[m] /= IndicesFrequencyMale[m + 1] - IndicesFrequencyMale[m];
|
||||
}
|
||||
|
||||
// Fem energy bin
|
||||
for (let m = 0; m < BoundingFrequencyFem.length - 1; m++) {
|
||||
for (
|
||||
let j = IndicesFrequencyMale[m];
|
||||
j <= IndicesFrequencyMale[m + 1];
|
||||
j++
|
||||
)
|
||||
if (sensitivityPerPole[j] > 0) EnergyBinFem[m] += sensitivityPerPole[j];
|
||||
EnergyBinMasc[m] /= IndicesFrequencyMale[m + 1] - IndicesFrequencyMale[m];
|
||||
EnergyBinFem[m] =
|
||||
EnergyBinFem[m] /
|
||||
(IndicesFrequencyFemale[m + 1] - IndicesFrequencyFemale[m]);
|
||||
}
|
||||
const oh =
|
||||
Math.max(EnergyBinFem[1], EnergyBinMasc[1]) > 0.2
|
||||
? 1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])
|
||||
: (1 - 2 * Math.max(EnergyBinMasc[2], EnergyBinFem[2])) *
|
||||
5 *
|
||||
Math.max(EnergyBinMasc[1], EnergyBinFem[1]);
|
||||
|
||||
const ah = 3 * Math.max(EnergyBinMasc[3], EnergyBinFem[3]);
|
||||
const ee =
|
||||
0.8 *
|
||||
(Math.max(EnergyBinMasc[1], EnergyBinFem[1]) -
|
||||
Math.max(EnergyBinMasc[3], EnergyBinFem[3]));
|
||||
return { oh, ee, ah };
|
||||
}
|
||||
|
||||
static createAudioMeter(audioContext) {
|
||||
const processor = audioContext.createScriptProcessor(512);
|
||||
processor.onaudioprocess = (event) => {
|
||||
const buf = event.inputBuffer.getChannelData(0);
|
||||
const bufLength = buf.length;
|
||||
let sum = 0;
|
||||
let x;
|
||||
|
||||
// eslint-disable-next-line no-plusplus
|
||||
for (let i = 0; i < bufLength; i++) {
|
||||
x = buf[i];
|
||||
if (Math.abs(x) >= processor.clipLevel) {
|
||||
processor.clipping = true;
|
||||
processor.lastClip = window.performance.now();
|
||||
}
|
||||
sum += x * x;
|
||||
}
|
||||
const rms = Math.sqrt(sum / bufLength);
|
||||
processor.volume = Math.max(rms, processor.volume * processor.averaging);
|
||||
};
|
||||
processor.clipping = false;
|
||||
processor.lastClip = 0;
|
||||
processor.volume = 0;
|
||||
processor.clipLevel = 0.98;
|
||||
processor.averaging = 0.95;
|
||||
processor.clipLag = 750;
|
||||
|
||||
processor.connect(audioContext.destination);
|
||||
|
||||
processor.checkClipping = () => {
|
||||
if (!processor.clipping) {
|
||||
return false;
|
||||
}
|
||||
if (processor.lastClip + processor.clipLag < window.performance.now()) {
|
||||
processor.clipping = false;
|
||||
}
|
||||
return processor.clipping;
|
||||
};
|
||||
|
||||
processor.shutdown = () => {
|
||||
processor.disconnect();
|
||||
processor.onaudioprocess = null;
|
||||
};
|
||||
|
||||
return processor;
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import { TextGeometry } from "three/addons/geometries/TextGeometry.js";
|
||||
import { FontLoader } from "three/examples/jsm/loaders/FontLoader.js";
|
||||
import { Font } from "three/examples/jsm/loaders/FontLoader.js";
|
||||
|
||||
export class LoadingPlaceHolder3DModel {
|
||||
public message_mesh: THREE.Mesh | undefined;
|
||||
|
||||
constructor() {
|
||||
this.message_mesh = undefined;
|
||||
}
|
||||
|
||||
public async destroyWithScene(scene: THREE.Scene) {
|
||||
if (this.message_mesh === undefined) {
|
||||
return;
|
||||
}
|
||||
scene.remove(this.message_mesh);
|
||||
this.message_mesh.geometry.dispose();
|
||||
}
|
||||
|
||||
public async initialize(
|
||||
message: string,
|
||||
position: THREE.Vector3,
|
||||
font: Font,
|
||||
): Promise<THREE.Mesh> {
|
||||
return (this.message_mesh = await this.createTextGeometry(
|
||||
message,
|
||||
position,
|
||||
font,
|
||||
));
|
||||
}
|
||||
|
||||
private async createTextGeometry(
|
||||
text: string,
|
||||
position: THREE.Vector3,
|
||||
font: Font,
|
||||
): Promise<THREE.Mesh> {
|
||||
if (font) {
|
||||
const textGeometry = new TextGeometry(text, {
|
||||
font: font,
|
||||
size: 0.25,
|
||||
height: 0.1,
|
||||
curveSegments: 1,
|
||||
bevelEnabled: false,
|
||||
bevelThickness: 1,
|
||||
bevelSize: 1,
|
||||
bevelOffset: 1,
|
||||
bevelSegments: 5,
|
||||
});
|
||||
|
||||
const textMaterial = new THREE.MeshPhongMaterial({ color: 0xffffff });
|
||||
const message_mesh = new THREE.Mesh(textGeometry, textMaterial);
|
||||
message_mesh.position.set(position.x, position.y, position.z);
|
||||
return message_mesh;
|
||||
}
|
||||
throw Error(`Could not Create Text Geometry for ${text}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class LoadingPlaceHolderManager {
|
||||
private messages: Map<string, LoadingPlaceHolder3DModel>;
|
||||
private font: Font | undefined;
|
||||
private scene: THREE.Scene;
|
||||
|
||||
constructor(scene: THREE.Scene) {
|
||||
this.messages = new Map<string, LoadingPlaceHolder3DModel>();
|
||||
this.font = undefined;
|
||||
this.scene = scene;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
this.font = await this.loadFont();
|
||||
}
|
||||
|
||||
public async remove(id: string) {
|
||||
const result = this.messages.get(id);
|
||||
if (result === undefined) {
|
||||
return;
|
||||
}
|
||||
result.destroyWithScene(this.scene);
|
||||
this.messages.delete(id);
|
||||
}
|
||||
|
||||
public async add(id: string, message: string, position: THREE.Vector3) {
|
||||
const placeholder = new LoadingPlaceHolder3DModel();
|
||||
if (this.font === undefined) {
|
||||
console.log("Forgot to Initialize LoadingPlaceHolderManager");
|
||||
return;
|
||||
}
|
||||
const placeholder3DObj = await placeholder.initialize(
|
||||
message,
|
||||
position,
|
||||
this.font,
|
||||
);
|
||||
this.scene.add(placeholder3DObj);
|
||||
this.messages.set(id, placeholder);
|
||||
}
|
||||
|
||||
// Load the font wrapper
|
||||
private async loadFont(): Promise<Font> {
|
||||
return new Promise<Font>((resolve, reject) => {
|
||||
const loader = new FontLoader();
|
||||
loader.load(
|
||||
"/resources/fonts/CarterOneRegular.json",
|
||||
(font) => {
|
||||
resolve(font);
|
||||
},
|
||||
(progress) => {
|
||||
console.log(`Font Loaded: ${progress.loaded / progress.total}`);
|
||||
},
|
||||
(error) => {
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { MediaUploadApi, VideoApi } from "~/Classes/ApiManager";
|
||||
|
||||
import {
|
||||
ProcessStatus,
|
||||
StreamingProgressResponse,
|
||||
VideoAudioPreProcessor,
|
||||
} from "./VideoProcessor/video_audio_preprocessor";
|
||||
|
||||
class PreviewVideoGeneration {
|
||||
mediaUploadAPI: MediaUploadApi;
|
||||
videoAPI: VideoApi;
|
||||
videoAndAudioPreProcessor: VideoAudioPreProcessor;
|
||||
|
||||
onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void;
|
||||
|
||||
constructor(
|
||||
onProgress: (response: StreamingProgressResponse<ProcessStatus>) => void,
|
||||
) {
|
||||
this.onProgress = onProgress;
|
||||
this.videoAndAudioPreProcessor = new VideoAudioPreProcessor(onProgress);
|
||||
this.mediaUploadAPI = new MediaUploadApi();
|
||||
this.videoAPI = new VideoApi();
|
||||
}
|
||||
|
||||
// Cancel any part of the processing
|
||||
public async cancel() {}
|
||||
|
||||
private async stageOneProcessing() {}
|
||||
|
||||
private async stageTwoProcessing() {}
|
||||
|
||||
private async processVideo() {
|
||||
// check if video is cached or not if not jump to 50%
|
||||
// collect frames and run preprocessing in the background.
|
||||
// start polling for percentage change and convert and remap to remainint %
|
||||
}
|
||||
|
||||
async clearFrameBuffers() {}
|
||||
}
|
||||
|
||||
export { PreviewVideoGeneration };
|
||||
@@ -1,97 +0,0 @@
|
||||
export class MoveAIResult {
|
||||
bone: string;
|
||||
is_special: boolean;
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
flip: boolean;
|
||||
only_y: boolean;
|
||||
|
||||
constructor(bone: string, is_special: boolean = false, x: number = 0, y: number = 0, z: number = 0,
|
||||
flip:boolean=false, only_y:boolean=false) {
|
||||
this.bone = bone;
|
||||
this.is_special = is_special;
|
||||
this.x = x;
|
||||
this.y = y;
|
||||
this.z = z;
|
||||
this.flip = flip;
|
||||
this.only_y = only_y;
|
||||
}
|
||||
}
|
||||
|
||||
export class Retarget {
|
||||
move_ai_options: { [Key: string]: MoveAIResult }
|
||||
constructor() {
|
||||
this.move_ai_options = {
|
||||
"_1Head.quaternion": new MoveAIResult("mixamorigHead.quaternion"),
|
||||
"_1Neck.quaternion": new MoveAIResult("mixamorigNeck.quaternion"),
|
||||
"_Spine.quaternion": new MoveAIResult("mixamorigSpine.quaternion"),
|
||||
"_1Spine.quaternion": new MoveAIResult("mixamorigSpine1.quaternion"),
|
||||
"_3Spine.quaternion": new MoveAIResult("mixamorigSpine2.quaternion"),
|
||||
|
||||
"_1Hips.quaternion": new MoveAIResult("mixamorigHips.quaternion", true, -90, 0, 0, false, true),
|
||||
"_1Hips.position": new MoveAIResult("mixamorigHips.position", true, 1, 0.05, 1),
|
||||
|
||||
"_1LeftFoot.quaternion": new MoveAIResult("mixamorigLeftFoot.quaternion", true, 0, 0, 0, true),
|
||||
"_1RightFoot.quaternion": new MoveAIResult("mixamorigRightFoot.quaternion", true, 0, 0, 0, true),
|
||||
|
||||
|
||||
"_1RightShoulder.quaternion": new MoveAIResult("mixamorigRightShoulder.quaternion", true, 0, 0, 0, false),
|
||||
"_1LeftShoulder.quaternion": new MoveAIResult("mixamorigLeftShoulder.quaternion", true, 0, 0, 0, false),
|
||||
"_1LeftArm.quaternion": new MoveAIResult("mixamorigLeftArm.quaternion", true, 0, 0, 0, false),
|
||||
"_1RightArm.quaternion": new MoveAIResult("mixamorigRightArm.quaternion", true, 0, 0, 0, false),
|
||||
"_1LeftForeArm.quaternion": new MoveAIResult("mixamorigLeftForeArm.quaternion"),
|
||||
"_1RightForeArm.quaternion": new MoveAIResult("mixamorigRightForeArm.quaternion"),
|
||||
|
||||
"_1LeftLeg.quaternion": new MoveAIResult("mixamorigLeftLeg.quaternion", true, 0, 0, 0, true),
|
||||
"_1RightLeg.quaternion": new MoveAIResult("mixamorigRightLeg.quaternion", true, 0, 0, 0, true),
|
||||
|
||||
"_1LeftUpLeg.quaternion": new MoveAIResult("mixamorigLeftUpLeg.quaternion", true, 0, 180, 0),
|
||||
"_1RightUpLeg.quaternion": new MoveAIResult("mixamorigRightUpLeg.quaternion", true, 0, 180, 0),
|
||||
|
||||
"_1LeftHand.quaternion": new MoveAIResult("mixamorigLeftHand.quaternion"),
|
||||
"_1RightHand.quaternion": new MoveAIResult("mixamorigRightHand.quaternion"),
|
||||
|
||||
"_1LeftHandThumb1.quaternion": new MoveAIResult("mixamorigLeftHandThumb1.quaternion"),
|
||||
"_1LeftHandThumb2.quaternion": new MoveAIResult("mixamorigLeftHandThumb2.quaternion"),
|
||||
"_1LeftHandThumb3.quaternion": new MoveAIResult("mixamorigLeftHandThumb3.quaternion"),
|
||||
"_1LeftHandIndex1.quaternion": new MoveAIResult("mixamorigLeftHandIndex1.quaternion"),
|
||||
"_1LeftHandIndex2.quaternion": new MoveAIResult("mixamorigLeftHandIndex2.quaternion"),
|
||||
"_1LeftHandIndex3.quaternion": new MoveAIResult("mixamorigLeftHandIndex3.quaternion"),
|
||||
"_1LeftHandMiddle1.quaternion": new MoveAIResult("mixamorigLeftHandMiddle1.quaternion"),
|
||||
"_1LeftHandMiddle2.quaternion": new MoveAIResult("mixamorigLeftHandMiddle2.quaternion"),
|
||||
"_1LeftHandMiddle3.quaternion": new MoveAIResult("mixamorigLeftHandMiddle3.quaternion"),
|
||||
"_1LeftHandRing1.quaternion": new MoveAIResult("mixamorigLeftHandRing1.quaternion"),
|
||||
"_1LeftHandRing2.quaternion": new MoveAIResult("mixamorigLeftHandRing2.quaternion"),
|
||||
"_1LeftHandRing3.quaternion": new MoveAIResult("mixamorigLeftHandRing3.quaternion"),
|
||||
"_1LeftHandPinky1.quaternion": new MoveAIResult("mixamorigLeftHandPinky1.quaternion"),
|
||||
"_1LeftHandPinky2.quaternion": new MoveAIResult("mixamorigLeftHandPinky2.quaternion"),
|
||||
"_1LeftHandPinky3.quaternion": new MoveAIResult("mixamorigLeftHandPinky3.quaternion"),
|
||||
"_1RightHandThumb1.quaternion": new MoveAIResult("mixamorigRightHandThumb1.quaternion"),
|
||||
"_1RightHandThumb2.quaternion": new MoveAIResult("mixamorigRightHandThumb2.quaternion"),
|
||||
"_1RightHandThumb3.quaternion": new MoveAIResult("mixamorigRightHandThumb3.quaternion"),
|
||||
"_1RightHandIndex1.quaternion": new MoveAIResult("mixamorigRightHandIndex1.quaternion"),
|
||||
"_1RightHandIndex2.quaternion": new MoveAIResult("mixamorigRightHandIndex2.quaternion"),
|
||||
"_1RightHandIndex3.quaternion": new MoveAIResult("mixamorigRightHandIndex3.quaternion"),
|
||||
"_1RightHandMiddle1.quaternion": new MoveAIResult("mixamorigRightHandMiddle1.quaternion"),
|
||||
"_1RightHandMiddle2.quaternion": new MoveAIResult("mixamorigRightHandMiddle2.quaternion"),
|
||||
"_1RightHandMiddle3.quaternion": new MoveAIResult("mixamorigRightHandMiddle3.quaternion"),
|
||||
"_1RightHandRing1.quaternion": new MoveAIResult("mixamorigRightHandRing1.quaternion"),
|
||||
"_1RightHandRing2.quaternion": new MoveAIResult("mixamorigRightHandRing2.quaternion"),
|
||||
"_1RightHandRing3.quaternion": new MoveAIResult("mixamorigRightHandRing3.quaternion"),
|
||||
"_1RightHandPinky1.quaternion": new MoveAIResult("mixamorigRightHandPinky1.quaternion"),
|
||||
"_1RightHandPinky2.quaternion": new MoveAIResult("mixamorigRightHandPinky2.quaternion"),
|
||||
"_1RightHandPinky3.quaternion": new MoveAIResult("mixamorigRightHandPinky3.quaternion"),
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
retarget(boneName: string): MoveAIResult {
|
||||
for (const [key, value] of Object.entries(this.move_ai_options)) {
|
||||
if (boneName == key) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return new MoveAIResult(boneName);
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
// @see https://discourse.threejs.org/t/production-ready-green-screen-with-three-js/23113/2
|
||||
|
||||
const VERTEX_SHADER = `
|
||||
varying vec2 vUv;
|
||||
|
||||
void main() {
|
||||
vUv = uv;
|
||||
gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
|
||||
}
|
||||
`;
|
||||
const FRAGMENT_SHADER = `
|
||||
uniform sampler2D tex;
|
||||
uniform float texWidth;
|
||||
uniform float texHeight;
|
||||
|
||||
uniform vec3 keyColor;
|
||||
uniform float similarity;
|
||||
uniform float smoothness;
|
||||
uniform float spill;
|
||||
|
||||
varying vec2 vUv;
|
||||
|
||||
// From https://github.com/libretro/glsl-shaders/blob/master/nnedi3/shaders/rgb-to-yuv.glsl
|
||||
vec2 RGBtoUV(vec3 rgb) {
|
||||
return vec2(
|
||||
rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,
|
||||
rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5
|
||||
);
|
||||
}
|
||||
|
||||
vec4 ProcessChromaKey(vec2 texCoord) {
|
||||
vec4 rgba = texture2D(tex, texCoord);
|
||||
float chromaDist = distance(RGBtoUV(texture2D(tex, texCoord).rgb), RGBtoUV(keyColor));
|
||||
|
||||
float baseMask = chromaDist - similarity;
|
||||
float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5);
|
||||
rgba.a = fullMask;
|
||||
|
||||
float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5);
|
||||
float desat = clamp(rgba.r * 0.2126 + rgba.g * 0.7152 + rgba.b * 0.0722, 0., 1.);
|
||||
rgba.rgb = mix(vec3(desat, desat, desat), rgba.rgb, spillVal);
|
||||
|
||||
return rgba;
|
||||
}
|
||||
|
||||
void main(void) {
|
||||
vec2 texCoord = vUv;
|
||||
gl_FragColor = ProcessChromaKey(texCoord);
|
||||
}
|
||||
`;
|
||||
|
||||
export { VERTEX_SHADER, FRAGMENT_SHADER };
|
||||
@@ -1,950 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
|
||||
import { ClipUI } from "../clips/clip_ui";
|
||||
|
||||
import Scene from "./scene.js";
|
||||
import AudioEngine from "./Engines/audio_engine";
|
||||
import TransformEngine from "./Engines/transform_engine";
|
||||
import LipSyncEngine from "./Engines/lip_sync_engine";
|
||||
|
||||
import Queue, {
|
||||
UnionedActionTypes,
|
||||
UnionedDataTypes,
|
||||
} from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "../Queue/QueueNames";
|
||||
import { toEngineActions } from "../Queue/toEngineActions";
|
||||
import { fromEngineActions } from "../Queue/fromEngineActions";
|
||||
import { ClipGroup, ClipType, AssetType } from "~/enums";
|
||||
import { CameraAspectRatio, MediaFileType } from "~/pages/PageEnigma/enums";
|
||||
import { Keyframe, MediaItem, UpdateTime } from "~/pages/PageEnigma/models";
|
||||
import Editor from "~/pages/PageEnigma/Editor/editor";
|
||||
import EmotionEngine from "./Engines/emotion_engine";
|
||||
import { IGenerationOptions } from "~/pages/PageEnigma/models/generationOptions";
|
||||
import { Vector3 } from "three";
|
||||
|
||||
import { filmLength, outlinerState } from "../signals";
|
||||
import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
|
||||
import { SplatMesh } from "@sparkjsdev/spark";
|
||||
|
||||
export class TimeLine {
|
||||
editorEngine: Editor;
|
||||
timeline_items: ClipUI[];
|
||||
|
||||
timeline_limit: number;
|
||||
absolute_end: number;
|
||||
scrubber_frame_position: number;
|
||||
is_playing: boolean;
|
||||
is_repeating: boolean = true;
|
||||
|
||||
// plays audio
|
||||
audio_engine: AudioEngine;
|
||||
// key framing
|
||||
transform_engine: TransformEngine;
|
||||
// animation engine
|
||||
animation_engine: CharacterAnimationEngine;
|
||||
// lip sync engine
|
||||
lipSync_engine: LipSyncEngine;
|
||||
// emotion engine
|
||||
emotion_engine: EmotionEngine;
|
||||
|
||||
// characters
|
||||
characters: { [key: string]: ClipGroup };
|
||||
|
||||
scene: Scene;
|
||||
camera: THREE.Camera | null;
|
||||
mouse: THREE.Vector2 | undefined;
|
||||
|
||||
current_time: number;
|
||||
|
||||
camera_name: string;
|
||||
// ensure that the elements are loaded first.
|
||||
|
||||
private debounce_generate_video: boolean;
|
||||
|
||||
constructor(
|
||||
editor: Editor,
|
||||
audio_engine: AudioEngine,
|
||||
transform_engine: TransformEngine,
|
||||
lipsync_engine: LipSyncEngine,
|
||||
animation_engine: CharacterAnimationEngine,
|
||||
emotion_engine: EmotionEngine,
|
||||
scene: Scene,
|
||||
camera: THREE.Camera | null,
|
||||
mouse: THREE.Vector2 | undefined,
|
||||
camera_name: string,
|
||||
) {
|
||||
this.editorEngine = editor;
|
||||
this.timeline_items = [];
|
||||
this.characters = {};
|
||||
this.absolute_end = 60 * 12;
|
||||
this.timeline_limit = this.getTimelineEnd();
|
||||
this.camera = camera;
|
||||
this.mouse = mouse;
|
||||
|
||||
this.is_playing = false;
|
||||
this.scrubber_frame_position = 0; // in frames into the tl
|
||||
|
||||
// this will be used to play the audio clips
|
||||
this.audio_engine = audio_engine;
|
||||
this.transform_engine = transform_engine;
|
||||
this.lipSync_engine = lipsync_engine;
|
||||
this.animation_engine = animation_engine;
|
||||
this.emotion_engine = emotion_engine;
|
||||
|
||||
this.scene = scene;
|
||||
this.debounce_generate_video = false;
|
||||
Queue.subscribe(
|
||||
QueueNames.TO_ENGINE,
|
||||
"engine",
|
||||
this.handleTimelineActions.bind(this),
|
||||
);
|
||||
|
||||
this.current_time = 0;
|
||||
this.camera_name = camera_name;
|
||||
}
|
||||
|
||||
public async updateUI() {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.FROM_ENGINE,
|
||||
action: fromEngineActions.UPDATE_TIME_LINE,
|
||||
data: this.timeline_items,
|
||||
});
|
||||
}
|
||||
|
||||
public async pushEvent(action: fromEngineActions, data: UnionedDataTypes) {
|
||||
//this.current_time += 0.75;
|
||||
Queue.publish({
|
||||
queueName: QueueNames.FROM_ENGINE,
|
||||
action: fromEngineActions.UPDATE_TIME,
|
||||
data: data,
|
||||
});
|
||||
}
|
||||
|
||||
public isCharacter(uuid: string): boolean {
|
||||
this.timeline_items.forEach((clip) => {
|
||||
if (clip.group == ClipGroup.CHARACTER)
|
||||
this.characters[clip.object_uuid] = ClipGroup.CHARACTER;
|
||||
});
|
||||
|
||||
let result: boolean = false;
|
||||
for (const key in this.characters) {
|
||||
if (key === uuid) {
|
||||
result = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async handleTimelineActions(data: {
|
||||
action: UnionedActionTypes;
|
||||
data: UnionedDataTypes;
|
||||
}) {
|
||||
const action = data.action;
|
||||
switch (action) {
|
||||
case toEngineActions.ADD_KEYFRAME:
|
||||
await this.addKeyFrame(data.data as Keyframe);
|
||||
break;
|
||||
case toEngineActions.UPDATE_KEYFRAME:
|
||||
await this.updateKeyFrame(data.data as Keyframe);
|
||||
break;
|
||||
case toEngineActions.DELETE_KEYFRAME:
|
||||
await this.deleteKeyFrame(data.data as Keyframe);
|
||||
break;
|
||||
case toEngineActions.ADD_CLIP:
|
||||
await this.addClip(data.data as ClipUI);
|
||||
break;
|
||||
case toEngineActions.DELETE_CLIP:
|
||||
await this.deleteClip(data.data as ClipUI);
|
||||
break;
|
||||
case toEngineActions.UPDATE_CLIP:
|
||||
await this.updateClip(data.data as ClipUI);
|
||||
break;
|
||||
case toEngineActions.UPDATE_TIME:
|
||||
await this.scrub(data.data as UpdateTime);
|
||||
break;
|
||||
case toEngineActions.MUTE:
|
||||
await this.mute(data.data as ClipUI, false);
|
||||
break;
|
||||
case toEngineActions.UNMUTE:
|
||||
await this.mute(data.data as ClipUI, true);
|
||||
break;
|
||||
// Create operations
|
||||
case toEngineActions.ADD_CHARACTER: {
|
||||
const newObject = await this.addCharacter(data.data as MediaItem);
|
||||
if (newObject)
|
||||
this.queueNewObjectMessage(
|
||||
newObject,
|
||||
data.data as MediaItem,
|
||||
AssetType.CHARACTER,
|
||||
);
|
||||
const result = this.editorEngine.sceneManager?.render_outliner(
|
||||
this.characters,
|
||||
);
|
||||
if (result) outlinerState.items.value = result.items;
|
||||
break;
|
||||
}
|
||||
case toEngineActions.ADD_OBJECT: {
|
||||
const newObject = await this.addObject(data.data as MediaItem);
|
||||
if (newObject)
|
||||
this.queueNewObjectMessage(
|
||||
newObject,
|
||||
data.data as MediaItem,
|
||||
newObject.type as AssetType
|
||||
);
|
||||
const result = this.editorEngine.sceneManager?.render_outliner(
|
||||
this.characters,
|
||||
);
|
||||
if (result) outlinerState.items.value = result.items;
|
||||
break;
|
||||
}
|
||||
case toEngineActions.ADD_SHAPE: {
|
||||
const newShape = await this.addShape(data.data as MediaItem);
|
||||
this.queueNewObjectMessage(
|
||||
newShape,
|
||||
data.data as MediaItem,
|
||||
AssetType.SHAPE,
|
||||
);
|
||||
const result = this.editorEngine.sceneManager?.render_outliner(
|
||||
this.characters,
|
||||
);
|
||||
if (result) outlinerState.items.value = result.items;
|
||||
break;
|
||||
}
|
||||
case toEngineActions.ENTER_PREVIEW_STATE:
|
||||
await this.editorEngine.switchPreview();
|
||||
break;
|
||||
case toEngineActions.ENTER_EDIT_STATE:
|
||||
this.editorEngine.switchEdit();
|
||||
break;
|
||||
case toEngineActions.TOGGLE_CAMERA_STATE:
|
||||
this.editorEngine.switchCameraView();
|
||||
break;
|
||||
case toEngineActions.TOGGLE_REPEATING:
|
||||
this.is_repeating = !this.is_repeating;
|
||||
break;
|
||||
case toEngineActions.REFRESH_PREVIEW:
|
||||
if (this.editorEngine.switchPreviewToggle) {
|
||||
await this.editorEngine.generateFrame();
|
||||
}
|
||||
break;
|
||||
case toEngineActions.GENERATE_VIDEO: {
|
||||
console.log("Calling Generate Video");
|
||||
// debounce generate video ...
|
||||
if (this.debounce_generate_video == false) {
|
||||
this.debounce_generate_video = true;
|
||||
const options = data.data; // super overloaded talk to the devs about this. TODO... refactor
|
||||
// pass this in ... rather than doing it implicitly ...
|
||||
this.editorEngine.generation_options = options as IGenerationOptions;
|
||||
|
||||
await this.editorEngine.generateVideo();
|
||||
this.debounce_generate_video = false;
|
||||
}
|
||||
console.log("Generate Video Press Event");
|
||||
break;
|
||||
}
|
||||
case toEngineActions.CHANGE_CAMERA_ASPECT_RATIO: {
|
||||
this.editorEngine.changeRenderCameraAspectRatio(
|
||||
data.data as CameraAspectRatio,
|
||||
);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
console.log("Action Not Wired", action);
|
||||
}
|
||||
}
|
||||
|
||||
public async addCharacter(data: MediaItem) {
|
||||
const media_id = data.media_id;
|
||||
const name = data.name;
|
||||
const pos = this.getPos();
|
||||
const new_data = { ...data };
|
||||
|
||||
console.debug('Adding to engine scene manager: ', media_id, name, pos);
|
||||
|
||||
const obj = await this.editorEngine.sceneManager?.create(
|
||||
media_id,
|
||||
name,
|
||||
pos,
|
||||
);
|
||||
|
||||
if (obj) {
|
||||
obj.userData["name"] = name;
|
||||
obj.name = name;
|
||||
obj.position.copy(pos);
|
||||
obj.userData.isCharacter = true;
|
||||
|
||||
console.debug("Added character object to timeline; object: ", obj);
|
||||
|
||||
// TODO(bt,2025-01-28): Hack for debugging only. Remove soon.
|
||||
(window as any).lastCharacter = obj;
|
||||
|
||||
// TODO(bt,2025-01-28): Hack for debugging only. Remove soon.
|
||||
const lastRig = obj.children[0];
|
||||
if (!!lastRig && lastRig.name.includes("gamerig")) {
|
||||
(window as any).lastRig = lastRig;
|
||||
}
|
||||
|
||||
const object_uuid = obj.uuid;
|
||||
|
||||
this.characters[object_uuid] = ClipGroup.CHARACTER; // TODO: Create a class to make the idea of a character.
|
||||
new_data["object_uuid"] = object_uuid;
|
||||
|
||||
Queue.publish({
|
||||
queueName: QueueNames.FROM_ENGINE,
|
||||
action: fromEngineActions.UPDATE_CHARACTER_ID,
|
||||
data: new_data,
|
||||
});
|
||||
|
||||
this.addPlayableClip(
|
||||
new ClipUI(
|
||||
data.version,
|
||||
ClipType.FAKE,
|
||||
ClipGroup.CHARACTER,
|
||||
"Default",
|
||||
media_id,
|
||||
obj.uuid,
|
||||
obj.uuid,
|
||||
name,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
obj.userData["media_file_type"],
|
||||
),
|
||||
);
|
||||
|
||||
await this.editorEngine.sceneManager?.add_creation_undostack(obj);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
queueNewObjectMessage(
|
||||
item: THREE.Object3D<THREE.Object3DEventMap>,
|
||||
data: MediaItem,
|
||||
asset_type: AssetType.OBJECT | AssetType.CHARACTER | AssetType.SHAPE | AssetType.SPLAT,
|
||||
) {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.FROM_ENGINE,
|
||||
action: fromEngineActions.ADD_OBJECT,
|
||||
data: {
|
||||
media_id: data.media_id,
|
||||
type: asset_type,
|
||||
name: item.name,
|
||||
object_uuid: item.uuid,
|
||||
version: 1,
|
||||
} as MediaItem,
|
||||
});
|
||||
|
||||
// this.addPlayableClip(
|
||||
// new ClipUI(
|
||||
// data["version"],
|
||||
// ClipType.FAKE,
|
||||
// ClipGroup.OBJECT,
|
||||
// "Default",
|
||||
// data.media_id,
|
||||
// item.uuid,
|
||||
// item.uuid,
|
||||
// item.name,
|
||||
// 0,
|
||||
// 0,
|
||||
// 0,
|
||||
// this.scene.get_object_by_uuid(item.uuid)?.userData[
|
||||
// "media_file_type"
|
||||
// ],
|
||||
// ),
|
||||
// );
|
||||
}
|
||||
|
||||
public getPos() {
|
||||
this.editorEngine.utils.removeTransformControls(true);
|
||||
const raycaster = new THREE.Raycaster();
|
||||
raycaster.layers.enable(0);
|
||||
raycaster.layers.enable(1);
|
||||
if (this.editorEngine.mouse && this.camera) {
|
||||
raycaster.setFromCamera(this.editorEngine.mouse, this.camera);
|
||||
const intersects = raycaster.intersectObjects(
|
||||
this.scene.scene.children,
|
||||
true,
|
||||
);
|
||||
if (intersects.length > 0) {
|
||||
return intersects[0].point;
|
||||
}
|
||||
}
|
||||
return new THREE.Vector3(0, 0, 0);
|
||||
}
|
||||
|
||||
public async addObject(data: MediaItem & { position?: { x: number; y: number; z: number } }) {
|
||||
const dataPos = data.position;
|
||||
const pos = dataPos
|
||||
? new THREE.Vector3(dataPos.x, dataPos.y, dataPos.z)
|
||||
: this.getPos();
|
||||
const media_id = data.media_id;
|
||||
const name = data.name;
|
||||
|
||||
const obj = await this.editorEngine.sceneManager?.create(
|
||||
media_id,
|
||||
name,
|
||||
pos,
|
||||
);
|
||||
if (obj) {
|
||||
obj.userData["name"] = name;
|
||||
obj.name = name;
|
||||
obj.position.copy(pos);
|
||||
|
||||
await this.editorEngine.sceneManager?.add_creation_undostack(obj);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
// This method is for local dev testing of objects
|
||||
// and bypasses the queue system, that's kinda redundant anyway
|
||||
public addLocalSplat(fileBytes: ArrayBuffer | Uint8Array, flipVertical: boolean = false) {
|
||||
const splatMesh = new SplatMesh({ fileBytes: fileBytes });
|
||||
splatMesh.name = `{SplatMesh-${Date.now()}}`;
|
||||
|
||||
if (flipVertical) {
|
||||
splatMesh.rotateX(Math.PI);
|
||||
}
|
||||
|
||||
this.editorEngine.sceneManager?.scene.scene.add(splatMesh);
|
||||
}
|
||||
|
||||
public async addShape(data: MediaItem) {
|
||||
const pos = this.getPos();
|
||||
const parim = await this.editorEngine.create_parim(data.media_id, pos);
|
||||
await this.editorEngine.sceneManager?.add_creation_undostack(parim);
|
||||
return parim;
|
||||
}
|
||||
|
||||
public async addKeyFrame(data: Keyframe) {
|
||||
// KeyFrame Object
|
||||
// version: number;
|
||||
// clip_uuid: string;
|
||||
// group: ClipGroup;
|
||||
// object_uuid?: string;
|
||||
// offset: number;
|
||||
// position: XYZ
|
||||
// rotation: XYZ;
|
||||
// scale: XYZ;
|
||||
// selected?: boolean;
|
||||
const data_json = data;
|
||||
const uuid = data_json.object_uuid;
|
||||
const keyframe_uuid = data_json.keyframe_uuid;
|
||||
|
||||
let object_name = this.scene.get_object_by_uuid(uuid)?.name;
|
||||
if (object_name === undefined) {
|
||||
object_name = "undefined";
|
||||
}
|
||||
|
||||
this.transform_engine.addFrame(
|
||||
uuid,
|
||||
data_json.offset,
|
||||
data_json.position as Vector3,
|
||||
data_json.rotation as Vector3,
|
||||
data_json.scale as Vector3,
|
||||
data_json.offset,
|
||||
data_json.keyframe_uuid,
|
||||
);
|
||||
|
||||
await this.addPlayableClip(
|
||||
new ClipUI(
|
||||
data_json["version"],
|
||||
ClipType.TRANSFORM,
|
||||
data_json["group"],
|
||||
object_name,
|
||||
"",
|
||||
keyframe_uuid,
|
||||
uuid,
|
||||
object_name,
|
||||
0,
|
||||
data_json["offset"],
|
||||
data_json["offset"],
|
||||
MediaFileType.None,
|
||||
),
|
||||
);
|
||||
|
||||
const point = this.scene.createPoint(
|
||||
data_json.position as Vector3,
|
||||
data_json.rotation as Vector3,
|
||||
data_json.scale as Vector3,
|
||||
data_json.keyframe_uuid,
|
||||
);
|
||||
if (this.editorEngine.camera_person_mode) {
|
||||
point.visible = false;
|
||||
}
|
||||
this.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public checkEditorCanPlay() {
|
||||
this.editorEngine.can_playback = this.getLastClipEnd() > 1;
|
||||
this.editorEngine.updateSelectedUI();
|
||||
}
|
||||
|
||||
// This method is NOT responsible for deletion of the object from scene or other editor parts
|
||||
// It is only responsible for deleting the object from the timeline and its corresponding elements
|
||||
public deleteObject(object: THREE.Object3D<THREE.Object3DEventMap>) {
|
||||
if (!object) {
|
||||
return
|
||||
}
|
||||
|
||||
if (object.name === this.camera_name) {
|
||||
return;
|
||||
}
|
||||
|
||||
const object_uuid = object.uuid;
|
||||
|
||||
this.timeline_items.forEach((element) => {
|
||||
if (
|
||||
element.type == ClipType.TRANSFORM &&
|
||||
element.object_uuid == object_uuid
|
||||
) {
|
||||
this.scene.deletePoint(element.clip_uuid);
|
||||
}
|
||||
});
|
||||
|
||||
// TODO: In the future object should have relations that can be deleted inside the engines
|
||||
// and the engines handle the deletion of objects.
|
||||
// Character Animation Engine already does this.
|
||||
this.timeline_items = this.timeline_items.filter(
|
||||
(element) => element.object_uuid !== object_uuid,
|
||||
);
|
||||
|
||||
this.animation_engine.removeCharacter(object);
|
||||
|
||||
// Update react land here.
|
||||
}
|
||||
|
||||
public async addClip(data: ClipUI) {
|
||||
const object_uuid = data.object_uuid;
|
||||
const media_id = data.media_id;
|
||||
const name = data.name;
|
||||
const group = data.group;
|
||||
const version = 1;
|
||||
const type = data.type;
|
||||
const offset = data.offset;
|
||||
const end_offset = data.length + offset;
|
||||
const object = this.scene.get_object_by_uuid(object_uuid);
|
||||
const object_name =
|
||||
object?.name ?? "undefined";
|
||||
const clip_uuid = data.clip_uuid;
|
||||
|
||||
switch (type) {
|
||||
case "animation":
|
||||
await this.animation_engine.addCharacterAnimationMedia(object!, media_id, data);
|
||||
this.animation_engine.evaluate(this.current_time, this.timeline_limit);
|
||||
break;
|
||||
case "transform":
|
||||
this.transform_engine.loadObject(object_uuid, data.length);
|
||||
break;
|
||||
case "expression":
|
||||
this.emotion_engine.loadClip(object_uuid, media_id);
|
||||
break;
|
||||
case "audio":
|
||||
if (group === "character") {
|
||||
this.lipSync_engine.load_object(object_uuid, media_id);
|
||||
// media id for this as well it can be downloaded
|
||||
this.addPlayableClip(
|
||||
new ClipUI(
|
||||
version,
|
||||
ClipType.AUDIO,
|
||||
ClipGroup.CHARACTER,
|
||||
name,
|
||||
media_id,
|
||||
clip_uuid,
|
||||
object_uuid,
|
||||
object_name,
|
||||
offset,
|
||||
end_offset,
|
||||
0, // length
|
||||
this.scene.get_object_by_uuid(object_uuid)?.userData[
|
||||
"media_file_type"
|
||||
],
|
||||
),
|
||||
);
|
||||
return;
|
||||
} else {
|
||||
this.audio_engine.loadClip(media_id);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// media id for this as well it can be downloaded
|
||||
this.addPlayableClip(
|
||||
new ClipUI(
|
||||
version,
|
||||
type,
|
||||
group,
|
||||
name,
|
||||
media_id,
|
||||
clip_uuid,
|
||||
object_uuid,
|
||||
object_name,
|
||||
offset,
|
||||
end_offset, // length
|
||||
0,
|
||||
this.scene.get_object_by_uuid(object_uuid)?.userData["media_file_type"],
|
||||
),
|
||||
);
|
||||
|
||||
this.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public async addSelfAnimationClip(
|
||||
data: ClipUI,
|
||||
animation_clip: THREE.AnimationClip,
|
||||
) {
|
||||
const object_uuid = data.object_uuid;
|
||||
const media_id = data.media_id;
|
||||
const name = data.name;
|
||||
const group = data.group;
|
||||
const version = 1;
|
||||
const type = data.type;
|
||||
const offset = data.offset;
|
||||
const end_offset = data.length + offset;
|
||||
const object = this.scene.get_object_by_uuid(object_uuid);
|
||||
const object_name =
|
||||
object?.name ?? "undefined";
|
||||
const clip_uuid = data.clip_uuid;
|
||||
|
||||
this.animation_engine.addCharacterAnimation(object!, animation_clip, data);
|
||||
|
||||
// media id for this as well it can be downloaded
|
||||
this.addPlayableClip(
|
||||
new ClipUI(
|
||||
version,
|
||||
type,
|
||||
group,
|
||||
name,
|
||||
media_id,
|
||||
clip_uuid,
|
||||
object_uuid,
|
||||
object_name,
|
||||
offset,
|
||||
end_offset, // length
|
||||
0,
|
||||
this.scene.get_object_by_uuid(object_uuid)?.userData["media_file_type"],
|
||||
),
|
||||
);
|
||||
|
||||
this.checkEditorCanPlay();
|
||||
this.updateUI();
|
||||
}
|
||||
|
||||
public async deleteKeyFrame(data: Keyframe) {
|
||||
const keyframe_uuid = data.keyframe_uuid;
|
||||
const object_uuid = data.object_uuid;
|
||||
this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid);
|
||||
this.scene.deletePoint(keyframe_uuid);
|
||||
for (const element of this.timeline_items) {
|
||||
if (
|
||||
element.clip_uuid === keyframe_uuid &&
|
||||
element.object_uuid === object_uuid
|
||||
) {
|
||||
this.timeline_items = this.timeline_items.filter(
|
||||
(element) =>
|
||||
!(
|
||||
element.clip_uuid === keyframe_uuid &&
|
||||
element.object_uuid === object_uuid
|
||||
),
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
this.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public async updateKeyFrame(data: Keyframe) {
|
||||
const keyframe_uuid = data.keyframe_uuid;
|
||||
const keyframe_offset = data.offset;
|
||||
const object_uuid = data.object_uuid;
|
||||
|
||||
const keyframe_pos = data.position;
|
||||
const keyframe_rot = data.rotation;
|
||||
const keyframe_scl = data.scale;
|
||||
|
||||
this.transform_engine.clips[object_uuid].setOffset(
|
||||
keyframe_uuid,
|
||||
keyframe_offset,
|
||||
);
|
||||
this.transform_engine.clips[object_uuid].setTransform(
|
||||
keyframe_uuid,
|
||||
keyframe_pos as Vector3,
|
||||
keyframe_rot as Vector3,
|
||||
keyframe_scl as Vector3,
|
||||
);
|
||||
this.scene.updatePoint(
|
||||
keyframe_uuid,
|
||||
keyframe_pos as Vector3,
|
||||
keyframe_rot as Vector3,
|
||||
);
|
||||
this.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public async updateClip(data: ClipUI) {
|
||||
// only length and offset changes here.
|
||||
const object_uuid = data.object_uuid;
|
||||
const media_id = data.media_id;
|
||||
const offset = data.offset;
|
||||
const length = data.length + offset;
|
||||
const clip_uuid = data.clip_uuid;
|
||||
|
||||
for (const element of this.timeline_items) {
|
||||
if (
|
||||
element.media_id === media_id &&
|
||||
element.object_uuid === object_uuid &&
|
||||
element.clip_uuid == clip_uuid
|
||||
) {
|
||||
element.length = length;
|
||||
element.offset = offset;
|
||||
}
|
||||
}
|
||||
this.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public async deleteClip(data: ClipUI) {
|
||||
const object_uuid = data.object_uuid;
|
||||
const media_id = data.media_id;
|
||||
const clip_uuid = data.clip_uuid;
|
||||
|
||||
// Remove the clip from the items array
|
||||
for (let i = 0; i < this.timeline_items.length; i++) {
|
||||
const element = this.timeline_items[i];
|
||||
if (
|
||||
element.media_id === media_id &&
|
||||
element.object_uuid === object_uuid &&
|
||||
element.clip_uuid == clip_uuid
|
||||
) {
|
||||
this.timeline_items.splice(i, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Once the clip is removed, we need to make sure the engines aren't tracking it either
|
||||
// TODO: Ideally the engines should just handle this individually and we check them based on clip type/group
|
||||
// This way we won't need to iterate over all timeline items
|
||||
if (data.type === ClipType.ANIMATION) {
|
||||
const object = this.scene.get_object_by_uuid(object_uuid)!;
|
||||
this.animation_engine.removeAnimation(object, data)
|
||||
}
|
||||
|
||||
this.checkEditorCanPlay();
|
||||
}
|
||||
|
||||
public async mute(data: ClipUI, isMute: boolean) {
|
||||
this.timeline_items.forEach((element) => {
|
||||
if (element.group === data.group) {
|
||||
element.should_play = isMute;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async addPlayableClip(clip: ClipUI): Promise<void> {
|
||||
this.timeline_items.push(clip);
|
||||
}
|
||||
|
||||
public async scrub(data: UpdateTime): Promise<void> {
|
||||
if (this.is_playing) {
|
||||
return;
|
||||
}
|
||||
const value = Math.floor(data.currentTime);
|
||||
await this.setScrubberPosition(value);
|
||||
this.current_time = value;
|
||||
await this.update();
|
||||
|
||||
if (this.editorEngine.switchPreviewToggle) {
|
||||
await this.editorEngine.generateFrame();
|
||||
}
|
||||
}
|
||||
|
||||
// public streaming events into the timeline from
|
||||
public async setScrubberPosition(offset: number) {
|
||||
this.scrubber_frame_position = offset; // in ms
|
||||
}
|
||||
|
||||
// should play from the clip that is closest to the to scrubber
|
||||
public async play(): Promise<void> {
|
||||
console.log(`Play - Starting Timeline`);
|
||||
this.is_playing = true;
|
||||
}
|
||||
|
||||
public async resetScene() {
|
||||
for (const element of this.timeline_items) {
|
||||
if (element.type === ClipType.TRANSFORM) {
|
||||
const object = this.scene.get_object_by_uuid(element.object_uuid);
|
||||
if (object && this.transform_engine.clips[element.object_uuid]) {
|
||||
this.transform_engine.clips[element.object_uuid].reset(object);
|
||||
}
|
||||
} else if (
|
||||
element.type === ClipType.AUDIO &&
|
||||
element.group !== ClipGroup.CHARACTER
|
||||
) {
|
||||
this.audio_engine.loadClip(element.media_id);
|
||||
this.audio_engine.stopClip(element.media_id);
|
||||
} else if (element.type === ClipType.ANIMATION) {
|
||||
this.animation_engine.stop();
|
||||
} else if (
|
||||
element.type === ClipType.AUDIO &&
|
||||
element.group === ClipGroup.CHARACTER
|
||||
) {
|
||||
this.lipSync_engine.clips[
|
||||
element.object_uuid + element.media_id
|
||||
].stop();
|
||||
this.lipSync_engine.clips[
|
||||
element.object_uuid + element.media_id
|
||||
].reset();
|
||||
} else if (element.type === ClipType.EXPRESSION) {
|
||||
const object = this.scene.get_object_by_uuid(element.object_uuid);
|
||||
if (object)
|
||||
this.emotion_engine.clips[
|
||||
element.object_uuid + element.media_id
|
||||
].reset(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public getLastClipEnd(): number {
|
||||
let longest = 0;
|
||||
for (const element of this.timeline_items) {
|
||||
if (longest < element.length) {
|
||||
longest = element.length;
|
||||
}
|
||||
}
|
||||
return longest;
|
||||
}
|
||||
|
||||
public getTimelineEnd(): number {
|
||||
return filmLength.peek() * 1000;
|
||||
}
|
||||
|
||||
// called by the editor update loop on each frame
|
||||
public async update(
|
||||
isRendering = false,
|
||||
delta_time: number = 0,
|
||||
): Promise<boolean> {
|
||||
//if (this.is_playing === false) return; // start and stop
|
||||
this.timeline_limit = this.getTimelineEnd();
|
||||
if (this.is_playing) {
|
||||
this.current_time += delta_time * 1000;
|
||||
this.pushEvent(fromEngineActions.UPDATE_TIME, {
|
||||
currentTime: this.current_time,
|
||||
});
|
||||
this.scrubber_frame_position = this.current_time;
|
||||
}
|
||||
|
||||
if (this.scrubber_frame_position <= 0) {
|
||||
for (const video_plane of this.scene.video_planes) {
|
||||
video_plane.currentTime = 0;
|
||||
}
|
||||
await this.resetScene();
|
||||
}
|
||||
|
||||
for (const video_plane of this.scene.video_planes) {
|
||||
// Caps to 10fps so that the buffering issue is solved and it plays i am not sure how to fix this.
|
||||
// TODO: Fix buffering and make 30 fps.
|
||||
const fixedPoint = 1;
|
||||
|
||||
let pb = parseFloat(
|
||||
(this.scrubber_frame_position / this.editorEngine.cap_fps).toFixed(
|
||||
fixedPoint,
|
||||
),
|
||||
);
|
||||
pb = parseFloat((pb % video_plane.duration).toFixed(fixedPoint));
|
||||
if (video_plane.currentTime !== pb) {
|
||||
video_plane.playbackRate = 6;
|
||||
video_plane.currentTime = pb;
|
||||
}
|
||||
}
|
||||
|
||||
//this.scrubber_frame_position += 1;
|
||||
//2. allow stopping.
|
||||
//3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline.
|
||||
|
||||
// Since the animation engine is newer we can just call an evaluation on it instead of having the timeline process it
|
||||
this.animation_engine.evaluate(this.current_time, this.timeline_limit);
|
||||
|
||||
// Iterate over the clips and play them as fit
|
||||
for (const element of this.timeline_items) {
|
||||
if (
|
||||
element.offset <= this.scrubber_frame_position &&
|
||||
this.scrubber_frame_position <= element.length &&
|
||||
element.should_play
|
||||
) {
|
||||
// run async
|
||||
// element.play()
|
||||
// remove the element from the list
|
||||
const object = this.scene.get_object_by_uuid(element.object_uuid);
|
||||
if (element.type === ClipType.TRANSFORM) {
|
||||
if (object && this.transform_engine.clips[element.object_uuid]) {
|
||||
this.transform_engine.clips[element.object_uuid].step(
|
||||
object,
|
||||
element.offset,
|
||||
this.scrubber_frame_position,
|
||||
this.scene,
|
||||
);
|
||||
element.length =
|
||||
this.transform_engine.clips[element.object_uuid].length;
|
||||
}
|
||||
} else if (
|
||||
element.type === ClipType.AUDIO &&
|
||||
element.group !== ClipGroup.CHARACTER &&
|
||||
this.is_playing &&
|
||||
!isRendering
|
||||
) {
|
||||
// if (this.scrubber_frame_position + 1 >= element.length) {
|
||||
// this.audio_engine.playClip(element.media_id);
|
||||
// } else {
|
||||
// await this.audio_engine.step(
|
||||
// element.object_uuid + element.media_id,
|
||||
// this.scrubber_frame_position, element.offset);
|
||||
// }
|
||||
this.audio_engine.playClip(element.media_id);
|
||||
await this.audio_engine.step(
|
||||
element.media_id,
|
||||
this.scrubber_frame_position,
|
||||
element.offset,
|
||||
);
|
||||
} else if (
|
||||
element.type === ClipType.AUDIO &&
|
||||
element.group === ClipGroup.CHARACTER &&
|
||||
this.is_playing
|
||||
) {
|
||||
if (object) {
|
||||
await this.lipSync_engine.clips[
|
||||
element.object_uuid + element.media_id
|
||||
].play(object);
|
||||
this.lipSync_engine.clips[
|
||||
element.object_uuid + element.media_id
|
||||
].step(this.scrubber_frame_position, element.offset);
|
||||
}
|
||||
} else if (element.type === ClipType.EXPRESSION) {
|
||||
if (object) {
|
||||
await this.emotion_engine.clips[
|
||||
object.uuid + element.media_id
|
||||
].step(this.scrubber_frame_position - element.offset, object);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
this.scrubber_frame_position >= this.timeline_limit &&
|
||||
this.is_playing
|
||||
) {
|
||||
await this.stop();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async stop(): Promise<void> {
|
||||
await this.resetScene();
|
||||
this.is_playing = false;
|
||||
console.log(`Stop - Stopping Timeline`);
|
||||
this.current_time = 0;
|
||||
this.update();
|
||||
this.pushEvent(fromEngineActions.UPDATE_TIME, {
|
||||
currentTime: this.current_time,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
import { create } from "zustand";
|
||||
import { CameraAspectRatio, EditorStates } from "./enums";
|
||||
import { AssetType, AssetFilterOption, ArtStyle } from "~/enums";
|
||||
import { styleList } from "./styleList";
|
||||
|
||||
export type SceneObjectKind = "object" | "character" | "shape";
|
||||
|
||||
export interface SceneObject {
|
||||
id: string;
|
||||
kind: SceneObjectKind;
|
||||
name: string;
|
||||
mediaId?: string;
|
||||
mediaToken?: string;
|
||||
}
|
||||
|
||||
export interface CameraEntry {
|
||||
id: string;
|
||||
label: string;
|
||||
focalLength: number;
|
||||
position: { x: number; y: number; z: number };
|
||||
rotation: { x: number; y: number; z: number };
|
||||
lookAt: { x: number; y: number; z: number };
|
||||
}
|
||||
|
||||
export interface FocalLengthDragging {
|
||||
isDragging: boolean;
|
||||
focalLength: number;
|
||||
}
|
||||
|
||||
export interface SelectedSceneObject {
|
||||
type: AssetType;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export type TransformMode = "move" | "rotate" | "scale";
|
||||
export type TransformSpace = "world" | "local";
|
||||
export type PoseMode = "select" | "edit";
|
||||
|
||||
export enum DomLevels {
|
||||
NONE = 0,
|
||||
INPUT = 1,
|
||||
PANEL = 2,
|
||||
DIALOGUE = 3,
|
||||
}
|
||||
|
||||
export interface HotkeyStatus {
|
||||
disabled: boolean;
|
||||
disabledBy: DomLevels;
|
||||
}
|
||||
|
||||
export interface EditorLoader {
|
||||
isShowing: boolean;
|
||||
message: string | undefined;
|
||||
}
|
||||
|
||||
const DEFAULT_CAMERAS: CameraEntry[] = [
|
||||
{
|
||||
id: "main",
|
||||
label: "Main View",
|
||||
focalLength: 17,
|
||||
position: { x: -2.5, y: 2.5, z: 2.5 },
|
||||
rotation: { x: 0, y: 0, z: 0 },
|
||||
lookAt: { x: 0, y: 0, z: 0 },
|
||||
},
|
||||
{
|
||||
id: "cam2",
|
||||
label: "Camera 2",
|
||||
focalLength: 10,
|
||||
position: { x: 0, y: 0.6, z: 1.5 },
|
||||
rotation: { x: 0, y: 0, z: 0 },
|
||||
lookAt: { x: 0, y: 0, z: 0 },
|
||||
},
|
||||
];
|
||||
|
||||
interface PageEnigmaState {
|
||||
// scene contents
|
||||
objects: SceneObject[];
|
||||
characters: SceneObject[];
|
||||
shapes: SceneObject[];
|
||||
selectedObject: SelectedSceneObject | null;
|
||||
|
||||
// cameras
|
||||
cameras: CameraEntry[];
|
||||
selectedCameraId: string;
|
||||
cameraAspectRatio: CameraAspectRatio;
|
||||
focalLengthDragging: FocalLengthDragging;
|
||||
cameraFilter: AssetFilterOption;
|
||||
|
||||
// editor mode
|
||||
editorState: EditorStates;
|
||||
transformMode: TransformMode;
|
||||
transformSpace: TransformSpace;
|
||||
selectedMode: string;
|
||||
poseMode: PoseMode;
|
||||
showPoseControls: boolean;
|
||||
gridVisible: boolean;
|
||||
ignoreKeyDelete: boolean;
|
||||
hotkeyStatus: HotkeyStatus;
|
||||
isPromptBoxFocused: boolean;
|
||||
|
||||
// layout / panels
|
||||
sidePanelWidth: number;
|
||||
sidePanelVisible: boolean;
|
||||
dndSidePanelWidth: number;
|
||||
assetModalVisible: boolean;
|
||||
assetModalVisibleDuringDrag: boolean;
|
||||
reopenAfterDrag: boolean;
|
||||
|
||||
// overlays
|
||||
editorLoader: EditorLoader;
|
||||
editorLetterBox: boolean;
|
||||
previewSrc: string;
|
||||
showErrorDialog: boolean;
|
||||
errorDialogTitle: string;
|
||||
errorDialogMessage: string;
|
||||
|
||||
// movies
|
||||
viewMyMovies: boolean;
|
||||
generateMovieId: string;
|
||||
|
||||
// generation / stylize
|
||||
generateProgress: number;
|
||||
selectedArtStyle: ArtStyle;
|
||||
upscale: boolean;
|
||||
faceDetail: boolean;
|
||||
styleStrength: number;
|
||||
lipSync: boolean;
|
||||
cinematic: boolean;
|
||||
enginePreProcessing: boolean;
|
||||
globalIPAMediaToken: string | null;
|
||||
adapterImage: string | null;
|
||||
positivePrompt: string;
|
||||
negativePrompt: string;
|
||||
isUserInputPositive: boolean;
|
||||
isUserInputNegative: boolean;
|
||||
showNegativePrompt: boolean;
|
||||
|
||||
// ----- actions -----
|
||||
|
||||
// scene
|
||||
addObject: (obj: SceneObject) => void;
|
||||
addCharacter: (obj: SceneObject) => void;
|
||||
addShape: (obj: SceneObject) => void;
|
||||
removeSceneObject: (id: string) => void;
|
||||
setSelectedObject: (sel: SelectedSceneObject | null) => void;
|
||||
resetScene: () => void;
|
||||
|
||||
// camera
|
||||
addCamera: (camera: CameraEntry) => void;
|
||||
updateCamera: (id: string, updates: Partial<CameraEntry>) => void;
|
||||
deleteCamera: (id: string) => void;
|
||||
setSelectedCameraId: (id: string) => void;
|
||||
setCameraAspectRatio: (ratio: CameraAspectRatio) => void;
|
||||
setFocalLengthDragging: (state: FocalLengthDragging) => void;
|
||||
setCameraFilter: (filter: AssetFilterOption) => void;
|
||||
|
||||
// editor mode
|
||||
setEditorState: (state: EditorStates) => void;
|
||||
setTransformMode: (mode: TransformMode) => void;
|
||||
setTransformSpace: (space: TransformSpace) => void;
|
||||
setSelectedMode: (mode: string) => void;
|
||||
setPoseMode: (mode: PoseMode) => void;
|
||||
setShowPoseControls: (visible: boolean) => void;
|
||||
setGridVisible: (visible: boolean) => void;
|
||||
setIgnoreKeyDelete: (ignore: boolean) => void;
|
||||
disableHotkeyInput: (level: DomLevels) => void;
|
||||
enableHotkeyInput: (level: DomLevels) => void;
|
||||
setIsPromptBoxFocused: (focused: boolean) => void;
|
||||
|
||||
// layout
|
||||
setSidePanelWidth: (px: number) => void;
|
||||
setSidePanelVisible: (visible: boolean) => void;
|
||||
setDndSidePanelWidth: (px: number) => void;
|
||||
setAssetModalVisible: (visible: boolean) => void;
|
||||
setAssetModalVisibleDuringDrag: (visible: boolean) => void;
|
||||
setReopenAfterDrag: (reopen: boolean) => void;
|
||||
|
||||
// overlays
|
||||
showEditorLoader: (message?: string) => void;
|
||||
hideEditorLoader: () => void;
|
||||
toggleEditorLetterBox: (next?: boolean) => void;
|
||||
setPreviewSrc: (src: string) => void;
|
||||
setErrorDialog: (title: string, message: string) => void;
|
||||
setShowErrorDialog: (show: boolean) => void;
|
||||
|
||||
// movies
|
||||
setViewMyMovies: (view: boolean) => void;
|
||||
setGenerateMovieId: (id: string) => void;
|
||||
|
||||
// generation / stylize
|
||||
setGenerateProgress: (n: number) => void;
|
||||
setSelectedArtStyle: (style: ArtStyle) => void;
|
||||
setUpscale: (v: boolean) => void;
|
||||
setFaceDetail: (v: boolean) => void;
|
||||
setStyleStrength: (n: number) => void;
|
||||
setLipSync: (v: boolean) => void;
|
||||
setCinematic: (v: boolean) => void;
|
||||
setEnginePreProcessing: (v: boolean) => void;
|
||||
setGlobalIPAMediaToken: (t: string | null) => void;
|
||||
setAdapterImage: (s: string | null) => void;
|
||||
setPositivePrompt: (p: string) => void;
|
||||
setNegativePrompt: (p: string) => void;
|
||||
setIsUserInputPositive: (v: boolean) => void;
|
||||
setIsUserInputNegative: (v: boolean) => void;
|
||||
setShowNegativePrompt: (v: boolean) => void;
|
||||
resetSceneGenerationMetadata: () => void;
|
||||
}
|
||||
|
||||
export const usePageEnigmaStore = create<PageEnigmaState>((set, get) => ({
|
||||
// initial state
|
||||
objects: [],
|
||||
characters: [],
|
||||
shapes: [],
|
||||
selectedObject: null,
|
||||
|
||||
cameras: DEFAULT_CAMERAS,
|
||||
selectedCameraId: "main",
|
||||
cameraAspectRatio: CameraAspectRatio.HORIZONTAL_3_2,
|
||||
focalLengthDragging: { isDragging: false, focalLength: 35 },
|
||||
cameraFilter: AssetFilterOption.ALL,
|
||||
|
||||
editorState: EditorStates.EDIT,
|
||||
transformMode: "move",
|
||||
transformSpace: "world",
|
||||
selectedMode: "move",
|
||||
poseMode: "select",
|
||||
showPoseControls: false,
|
||||
gridVisible: true,
|
||||
ignoreKeyDelete: false,
|
||||
hotkeyStatus: { disabled: false, disabledBy: DomLevels.NONE },
|
||||
isPromptBoxFocused: false,
|
||||
|
||||
sidePanelWidth: 0,
|
||||
sidePanelVisible: false,
|
||||
dndSidePanelWidth: -1,
|
||||
assetModalVisible: false,
|
||||
assetModalVisibleDuringDrag: true,
|
||||
reopenAfterDrag: false,
|
||||
|
||||
editorLoader: { isShowing: false, message: "Loading Editor Engine 🦊" },
|
||||
editorLetterBox: true,
|
||||
previewSrc: "",
|
||||
showErrorDialog: false,
|
||||
errorDialogTitle: "Error!",
|
||||
errorDialogMessage: "Something went wrong.",
|
||||
|
||||
viewMyMovies: false,
|
||||
generateMovieId: "",
|
||||
|
||||
generateProgress: -1,
|
||||
selectedArtStyle: styleList[0].type,
|
||||
upscale: false,
|
||||
faceDetail: true,
|
||||
styleStrength: 0.8,
|
||||
lipSync: false,
|
||||
cinematic: true,
|
||||
enginePreProcessing: false,
|
||||
globalIPAMediaToken: null,
|
||||
adapterImage: null,
|
||||
positivePrompt: "",
|
||||
negativePrompt: "",
|
||||
isUserInputPositive: false,
|
||||
isUserInputNegative: false,
|
||||
showNegativePrompt: false,
|
||||
|
||||
// scene actions
|
||||
addObject: (obj) =>
|
||||
set((s) => ({ objects: [...s.objects, obj] })),
|
||||
addCharacter: (obj) =>
|
||||
set((s) => ({ characters: [...s.characters, obj] })),
|
||||
addShape: (obj) =>
|
||||
set((s) => ({ shapes: [...s.shapes, obj] })),
|
||||
removeSceneObject: (id) =>
|
||||
set((s) => ({
|
||||
objects: s.objects.filter((o) => o.id !== id),
|
||||
characters: s.characters.filter((o) => o.id !== id),
|
||||
shapes: s.shapes.filter((o) => o.id !== id),
|
||||
selectedObject:
|
||||
s.selectedObject?.id === id ? null : s.selectedObject,
|
||||
})),
|
||||
setSelectedObject: (sel) => set({ selectedObject: sel }),
|
||||
resetScene: () =>
|
||||
set({ objects: [], characters: [], shapes: [], selectedObject: null }),
|
||||
|
||||
// camera actions
|
||||
addCamera: (camera) => set((s) => ({ cameras: [...s.cameras, camera] })),
|
||||
updateCamera: (id, updates) =>
|
||||
set((s) => ({
|
||||
cameras: s.cameras.map((c) => (c.id === id ? { ...c, ...updates } : c)),
|
||||
})),
|
||||
deleteCamera: (id) => {
|
||||
if (id === "main") return;
|
||||
set((s) => ({
|
||||
cameras: s.cameras.filter((c) => c.id !== id),
|
||||
selectedCameraId: s.selectedCameraId === id ? "main" : s.selectedCameraId,
|
||||
}));
|
||||
},
|
||||
setSelectedCameraId: (id) => set({ selectedCameraId: id }),
|
||||
setCameraAspectRatio: (ratio) => set({ cameraAspectRatio: ratio }),
|
||||
setFocalLengthDragging: (state) => set({ focalLengthDragging: state }),
|
||||
setCameraFilter: (filter) => set({ cameraFilter: filter }),
|
||||
|
||||
// editor mode actions
|
||||
setEditorState: (state) => set({ editorState: state }),
|
||||
setTransformMode: (mode) => set({ transformMode: mode }),
|
||||
setTransformSpace: (space) => set({ transformSpace: space }),
|
||||
setSelectedMode: (mode) => set({ selectedMode: mode }),
|
||||
setPoseMode: (mode) => set({ poseMode: mode }),
|
||||
setShowPoseControls: (visible) => set({ showPoseControls: visible }),
|
||||
setGridVisible: (visible) => set({ gridVisible: visible }),
|
||||
setIgnoreKeyDelete: (ignore) => set({ ignoreKeyDelete: ignore }),
|
||||
disableHotkeyInput: (level) => {
|
||||
const status = get().hotkeyStatus;
|
||||
if (status.disabled) {
|
||||
if (level > status.disabledBy) {
|
||||
set({ hotkeyStatus: { ...status, disabledBy: level } });
|
||||
}
|
||||
} else {
|
||||
set({ hotkeyStatus: { disabled: true, disabledBy: level } });
|
||||
}
|
||||
},
|
||||
enableHotkeyInput: (level) => {
|
||||
const status = get().hotkeyStatus;
|
||||
if (status.disabled && level >= status.disabledBy) {
|
||||
set({ hotkeyStatus: { disabled: false, disabledBy: DomLevels.NONE } });
|
||||
}
|
||||
},
|
||||
setIsPromptBoxFocused: (focused) => set({ isPromptBoxFocused: focused }),
|
||||
|
||||
// layout actions
|
||||
setSidePanelWidth: (px) => set({ sidePanelWidth: px }),
|
||||
setSidePanelVisible: (visible) => set({ sidePanelVisible: visible }),
|
||||
setDndSidePanelWidth: (px) => set({ dndSidePanelWidth: px }),
|
||||
setAssetModalVisible: (visible) => set({ assetModalVisible: visible }),
|
||||
setAssetModalVisibleDuringDrag: (visible) =>
|
||||
set({ assetModalVisibleDuringDrag: visible }),
|
||||
setReopenAfterDrag: (reopen) => set({ reopenAfterDrag: reopen }),
|
||||
|
||||
// overlays actions
|
||||
showEditorLoader: (message) =>
|
||||
set({ editorLoader: { isShowing: true, message } }),
|
||||
hideEditorLoader: () =>
|
||||
set((s) => ({
|
||||
editorLoader: { isShowing: false, message: s.editorLoader.message },
|
||||
})),
|
||||
toggleEditorLetterBox: (next) =>
|
||||
set((s) => ({
|
||||
editorLetterBox: next !== undefined ? next : !s.editorLetterBox,
|
||||
})),
|
||||
setPreviewSrc: (src) => set({ previewSrc: src }),
|
||||
setErrorDialog: (title, message) =>
|
||||
set({
|
||||
errorDialogTitle: title,
|
||||
errorDialogMessage: message,
|
||||
showErrorDialog: true,
|
||||
}),
|
||||
setShowErrorDialog: (show) => set({ showErrorDialog: show }),
|
||||
|
||||
// movies actions
|
||||
setViewMyMovies: (view) => set({ viewMyMovies: view }),
|
||||
setGenerateMovieId: (id) => set({ generateMovieId: id }),
|
||||
|
||||
// generation / stylize actions
|
||||
setGenerateProgress: (n) => set({ generateProgress: n }),
|
||||
setSelectedArtStyle: (style) => set({ selectedArtStyle: style }),
|
||||
setUpscale: (v) => set({ upscale: v }),
|
||||
setFaceDetail: (v) => set({ faceDetail: v }),
|
||||
setStyleStrength: (n) => set({ styleStrength: n }),
|
||||
setLipSync: (v) => set({ lipSync: v }),
|
||||
setCinematic: (v) => set({ cinematic: v }),
|
||||
setEnginePreProcessing: (v) => set({ enginePreProcessing: v }),
|
||||
setGlobalIPAMediaToken: (t) => set({ globalIPAMediaToken: t }),
|
||||
setAdapterImage: (s) => set({ adapterImage: s }),
|
||||
setPositivePrompt: (p) => set({ positivePrompt: p }),
|
||||
setNegativePrompt: (p) => set({ negativePrompt: p }),
|
||||
setIsUserInputPositive: (v) => set({ isUserInputPositive: v }),
|
||||
setIsUserInputNegative: (v) => set({ isUserInputNegative: v }),
|
||||
setShowNegativePrompt: (v) => set({ showNegativePrompt: v }),
|
||||
resetSceneGenerationMetadata: () =>
|
||||
set({
|
||||
positivePrompt: "",
|
||||
negativePrompt: "",
|
||||
isUserInputPositive: false,
|
||||
isUserInputNegative: false,
|
||||
showNegativePrompt: false,
|
||||
selectedArtStyle: styleList[0].type,
|
||||
upscale: false,
|
||||
faceDetail: false,
|
||||
styleStrength: 0.8,
|
||||
lipSync: false,
|
||||
cinematic: false,
|
||||
enginePreProcessing: false,
|
||||
}),
|
||||
}));
|
||||
@@ -1,127 +0,0 @@
|
||||
import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
|
||||
import { CameraAspectRatio } from "../enums";
|
||||
import {
|
||||
QueueClip,
|
||||
QueueKeyframe,
|
||||
UpdateTime,
|
||||
MediaItem,
|
||||
} from "~/pages/PageEnigma/models";
|
||||
import { toTimelineActions } from "./toTimelineActions";
|
||||
|
||||
import { ClipUI } from "../clips/clip_ui";
|
||||
import { IGenerationOptions } from "../models/generationOptions";
|
||||
import { ToastDataType } from "~/components";
|
||||
import { QueueNames } from "./QueueNames";
|
||||
|
||||
export type UnionedActionTypes =
|
||||
| fromEngineActions
|
||||
| toEngineActions
|
||||
| toTimelineActions;
|
||||
|
||||
export type UnionedDataTypes =
|
||||
| QueueClip
|
||||
| UpdateTime
|
||||
| QueueKeyframe
|
||||
| ClipUI[]
|
||||
| MediaItem
|
||||
| ToastDataType
|
||||
| IGenerationOptions
|
||||
| CameraAspectRatio
|
||||
| null;
|
||||
|
||||
export type QueueSubscribeType = {
|
||||
action: UnionedActionTypes;
|
||||
data: UnionedDataTypes;
|
||||
};
|
||||
|
||||
export type SubscribeHandler = (entry: {
|
||||
action: UnionedActionTypes;
|
||||
data: UnionedDataTypes;
|
||||
}) => void;
|
||||
|
||||
class Queue {
|
||||
private _queues: Record<
|
||||
QueueNames,
|
||||
Array<{
|
||||
action: UnionedActionTypes;
|
||||
data: UnionedDataTypes;
|
||||
}>
|
||||
> = {
|
||||
[QueueNames.FROM_ENGINE]: [],
|
||||
[QueueNames.TO_ENGINE]: [],
|
||||
[QueueNames.TO_TIMELINE]: []
|
||||
};
|
||||
|
||||
private _subscribers: Record<
|
||||
QueueNames,
|
||||
Map<
|
||||
string,
|
||||
SubscribeHandler
|
||||
>
|
||||
> = {
|
||||
[QueueNames.FROM_ENGINE]: new Map<string, SubscribeHandler>(),
|
||||
[QueueNames.TO_ENGINE]: new Map<string, SubscribeHandler>(),
|
||||
[QueueNames.TO_TIMELINE]: new Map<string, SubscribeHandler>(),
|
||||
};
|
||||
|
||||
// Enqueues actions to a specified queue
|
||||
public publish({
|
||||
queueName,
|
||||
action,
|
||||
data,
|
||||
}: {
|
||||
queueName: QueueNames;
|
||||
action: UnionedActionTypes;
|
||||
data: UnionedDataTypes;
|
||||
}) {
|
||||
// Queues for all QueueNames should be initialized above but this is a safeguard for future enum additions
|
||||
// Make sure the queue exists
|
||||
if (!this._queues[queueName]) {
|
||||
this._queues[queueName] = [];
|
||||
}
|
||||
|
||||
// Queue the task in the right queue
|
||||
this._queues[queueName].push({ action, data });
|
||||
|
||||
// TODO: Ideally the publisher shouldn't deal at all with the subscribers
|
||||
// It should only publish the events to the right queue
|
||||
// It also doesn't make sense to fire an event on publish because the operations are then dispersed
|
||||
// leading to potential race conditions
|
||||
// FIXME: Add some sort of queue loop using internal event and add queue locks using promises to serialize queue publishes
|
||||
if (this._subscribers[queueName]) {
|
||||
this._subscribers[queueName].forEach((subcribeHandler) =>
|
||||
subcribeHandler(this._queues[queueName][0]),
|
||||
);
|
||||
this._queues[queueName].shift();
|
||||
}
|
||||
}
|
||||
|
||||
public subscribe(
|
||||
queueName: QueueNames,
|
||||
id: string,
|
||||
onMessage: (entry: QueueSubscribeType) => void,
|
||||
) {
|
||||
|
||||
// Queues for all QueueNames should be initialized above but this is a safeguard for future enum additions
|
||||
// Make sure the queue exists
|
||||
if (!this._subscribers[queueName]) {
|
||||
this._subscribers[queueName] = new Map<string, SubscribeHandler>();
|
||||
}
|
||||
|
||||
// Set the handler mapped to the ID
|
||||
this._subscribers[queueName].set(id, onMessage);
|
||||
|
||||
while (this._queues[queueName]?.length) {
|
||||
this._subscribers[queueName].forEach((subscribeHandler) =>
|
||||
subscribeHandler(this._queues[queueName][0]),
|
||||
);
|
||||
this._queues[queueName].shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const queue = new Queue();
|
||||
|
||||
export default queue;
|
||||
@@ -1,5 +0,0 @@
|
||||
export enum QueueNames {
|
||||
FROM_ENGINE = "FROM_ENGINE",
|
||||
TO_ENGINE = "TO_ENGINE",
|
||||
TO_TIMELINE = "TO_TIMELINE",
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
export enum fromEngineActions {
|
||||
ADD_OBJECT = "add_object",
|
||||
CAMERA_ASPECT_RATIO_CHANGED = "camera_aspect_ratio_changed",
|
||||
// data = CameraAspectRatio
|
||||
DELETE_OBJECT = "delete_object",
|
||||
DESELECT_OBJECT = "deselect_object",
|
||||
RESET_TIMELINE = "reset_timeline",
|
||||
SELECT_OBJECT = "select_object",
|
||||
UPDATE_CHARACTER_ID = "update_character_id",
|
||||
UPDATE_TIME = "update_time",
|
||||
UPDATE_TIME_LINE = "update_timeline",
|
||||
POP_A_TOAST = "pop_a_toast",
|
||||
// data={currentTime: number}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
import Queue from "./Queue";
|
||||
export default Queue;
|
||||
export * from "./QueueNames";
|
||||
@@ -1,47 +0,0 @@
|
||||
export enum toEngineActions {
|
||||
ADD_CLIP = "add_clip",
|
||||
// data=QueueClip
|
||||
ADD_KEYFRAME = "add_keyframe",
|
||||
|
||||
ADD_CHARACTER = "add_character",
|
||||
|
||||
ADD_OBJECT = "add_object",
|
||||
|
||||
ADD_SHAPE = "add_shape",
|
||||
|
||||
// data= QueueKeyframe
|
||||
DELETE_CLIP = "delete_clip",
|
||||
// data=QueueClip
|
||||
DELETE_KEYFRAME = "delete_keyframe",
|
||||
// data=QueueKeyframe
|
||||
ENTER_EDIT_STATE = "enter_edit",
|
||||
// data=null
|
||||
ENTER_PREVIEW_STATE = "enter_preview",
|
||||
// data=null
|
||||
GENERATE_VIDEO = "generate_video",
|
||||
// data=null
|
||||
MUTE = "mute",
|
||||
// data={version, type, group, object_uuid} if lipsync
|
||||
// data={version, type, group} if global_audio
|
||||
REFRESH_PREVIEW = "refresh_preview",
|
||||
// data=QueueClip
|
||||
TOGGLE_CAMERA_STATE = "toggle_camera",
|
||||
// data=null
|
||||
TOGGLE_REPEATING = "toggle_repeating",
|
||||
// data=null
|
||||
|
||||
CHANGE_CAMERA_ASPECT_RATIO = "change_camera_aspect_ratio",
|
||||
// data=CamperaAspectRatio
|
||||
|
||||
CAMERA_CHANGED = "camera_changed",
|
||||
|
||||
UNMUTE = "unmute",
|
||||
// data={version, type, group, object_uuid} if lipsync
|
||||
// data={version, type, group} if global_audio
|
||||
UPDATE_CLIP = "update_clip",
|
||||
// data=QueueClip
|
||||
UPDATE_KEYFRAME = "update_keyframe",
|
||||
// data=QueueKeyframe
|
||||
UPDATE_TIME = "update_time",
|
||||
// data={currentTime: number}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
export enum toInferenceActions {
|
||||
ADD_TTS_JOB = "add_tts_job"
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
export enum toTimelineActions {
|
||||
ADD_KEYFRAME = "add_keyframe",
|
||||
GENERATE_PROGRESS = "generate_progress",
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
export default interface IAnimationClip {
|
||||
speed: number;
|
||||
duration: number;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
evaluate: (time: number) => void;
|
||||
}
|
||||
@@ -1,348 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import { GLTFLoader } from "three/addons/loaders/GLTFLoader.js";
|
||||
import {
|
||||
FBXLoader,
|
||||
MMDAnimationHelper,
|
||||
MMDAnimationHelperMixer,
|
||||
} from "three/examples/jsm/Addons.js";
|
||||
import { MMDLoader } from "three/addons/loaders/MMDLoader.js";
|
||||
|
||||
import { MoveAIResult, Retarget } from "../Editor/retargeting";
|
||||
import IAnimationClip from "./ClipInterfaces/IAnimationClip";
|
||||
import { GetCdnOrigin } from "~/api/GetCdnOrigin";
|
||||
import { StorytellerApiHostStore } from "@storyteller/api";
|
||||
|
||||
export class AnimationClip implements IAnimationClip {
|
||||
version: number;
|
||||
media_id: string; // comes from the server
|
||||
object_uuid: string;
|
||||
type: "animation";
|
||||
location: "glb" | "remote";
|
||||
speed: number;
|
||||
duration: number;
|
||||
startTime: number;
|
||||
length: number;
|
||||
clip_name: string;
|
||||
mixer: THREE.AnimationMixer | undefined;
|
||||
animation_clip: THREE.AnimationClip | undefined;
|
||||
clip_action: THREE.AnimationAction | undefined;
|
||||
special_properties: MoveAIResult[];
|
||||
retargeted: boolean;
|
||||
last_frame: number;
|
||||
|
||||
|
||||
private isMMD: boolean = false;
|
||||
private obj: MMDAnimationHelperMixer | undefined;
|
||||
private helper: MMDAnimationHelper | undefined;
|
||||
private mmd_anim_clip: THREE.AnimationClip | undefined;
|
||||
private pose: Object | undefined;
|
||||
private processingClip: boolean = false;
|
||||
private mmd_url: string = "";
|
||||
|
||||
constructor(
|
||||
version: number,
|
||||
media_id: string,
|
||||
location: "glb" | "remote",
|
||||
object_uuid: string,
|
||||
speed: number,
|
||||
length: number,
|
||||
clip_name: string,
|
||||
startTime: number,
|
||||
duration: number,
|
||||
) {
|
||||
this.version = version;
|
||||
this.media_id = media_id;
|
||||
this.type = "animation";
|
||||
this.object_uuid = object_uuid;
|
||||
this.location = location;
|
||||
this.speed = speed;
|
||||
this.length = length;
|
||||
this.startTime = startTime;
|
||||
this.duration = duration;
|
||||
this.clip_name = clip_name;
|
||||
this.animation_clip;
|
||||
this.mixer;
|
||||
this.clip_action;
|
||||
this.special_properties = [];
|
||||
this.retargeted = false;
|
||||
this.last_frame = 0;
|
||||
}
|
||||
|
||||
get endTime() {
|
||||
return this.startTime + this.duration
|
||||
}
|
||||
|
||||
evaluate: (time: number) => void;
|
||||
|
||||
async get_media_url() {
|
||||
//This is for prod when we have the proper info on the url.
|
||||
const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`;
|
||||
|
||||
const response = await fetch(url);
|
||||
const json = await JSON.parse(await response.text());
|
||||
const bucketPath = json["media_file"]["public_bucket_path"];
|
||||
|
||||
const media_api_base_url = GetCdnOrigin();
|
||||
|
||||
const media_url = `${media_api_base_url}${bucketPath}`;
|
||||
return media_url;
|
||||
}
|
||||
|
||||
_load_animation(): Promise<THREE.AnimationClip> {
|
||||
// Return the promise chain starting from `this.get_media_url()`
|
||||
return this.get_media_url().then((url) => {
|
||||
// Return a new Promise that resolves with the animation clip
|
||||
return new Promise((resolve) => {
|
||||
if (url.includes(".glb")) {
|
||||
const glbLoader = new GLTFLoader();
|
||||
|
||||
glbLoader.load(url, (gltf) => {
|
||||
// Assuming the animation is the first one in the animations array
|
||||
const animationClip = gltf.animations[0];
|
||||
resolve(animationClip);
|
||||
});
|
||||
} else if (url.includes(".fbx")) {
|
||||
const fbxLoader = new FBXLoader();
|
||||
fbxLoader.load(url, (fbx) => {
|
||||
const animationClip = fbx.animations[0];
|
||||
this.retargeted = true;
|
||||
animationClip.tracks.forEach((track) => {
|
||||
const retarget = new Retarget();
|
||||
const retarget_value = retarget.retarget(track.name);
|
||||
track.name = retarget_value.bone;
|
||||
console.log(track);
|
||||
if (retarget_value.is_special) {
|
||||
this.special_properties.push(retarget_value);
|
||||
}
|
||||
});
|
||||
resolve(animationClip);
|
||||
});
|
||||
} else if (url.includes(".vmd")) {
|
||||
console.log("VMD Loader");
|
||||
const root = this.mixer?.getRoot();
|
||||
this.isMMD = true;
|
||||
if (root) {
|
||||
const mmdLoader = new MMDLoader();
|
||||
this.mmd_url = url;
|
||||
mmdLoader.loadAnimation(url, root as THREE.SkinnedMesh, (mmd) => {
|
||||
mmd.name = this.mmd_url;
|
||||
resolve(mmd as THREE.AnimationClip);
|
||||
});
|
||||
console.log("Loaded");
|
||||
}
|
||||
console.log("Dont");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
_create_mixer(object: THREE.Object3D) {
|
||||
this.mixer = new THREE.AnimationMixer(object);
|
||||
return this.mixer;
|
||||
}
|
||||
|
||||
async _get_clip() {
|
||||
if (this.animation_clip == null && this.mixer !== null) {
|
||||
console.log(this.media_id);
|
||||
if (this.media_id === "SelfClip") {
|
||||
const rootObject = this.mixer?.getRoot();
|
||||
if (rootObject)
|
||||
this.animation_clip = (
|
||||
rootObject as THREE.Object3D<THREE.Object3DEventMap>
|
||||
).animations[0];
|
||||
} else {
|
||||
this.animation_clip = await this._load_animation();
|
||||
}
|
||||
}
|
||||
return this.animation_clip;
|
||||
}
|
||||
|
||||
async play(object: THREE.Object3D) {
|
||||
if (this.mixer == null) {
|
||||
this.mixer = this._create_mixer(object);
|
||||
const anim_clip = await this._get_clip();
|
||||
if (anim_clip == undefined) {
|
||||
return;
|
||||
}
|
||||
this.clip_action = this.mixer?.clipAction(anim_clip);
|
||||
if (this.clip_action && this.isMMD === false) {
|
||||
if (this.clip_action?.isRunning() == false) {
|
||||
this.clip_action.play();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
update_bones() {
|
||||
if (this.retargeted === false) {
|
||||
return;
|
||||
}
|
||||
const rootObject = this.mixer?.getRoot();
|
||||
if (rootObject)
|
||||
for (
|
||||
let index_ = 0;
|
||||
index_ <
|
||||
(rootObject as THREE.Object3D<THREE.Object3DEventMap>).children.length;
|
||||
index_++
|
||||
) {
|
||||
const child_holder = (
|
||||
rootObject as THREE.Object3D<THREE.Object3DEventMap>
|
||||
).children[index_];
|
||||
if (child_holder.type == "Bone") {
|
||||
child_holder.traverse(
|
||||
(bone: THREE.Object3D<THREE.Object3DEventMap>) => {
|
||||
for (
|
||||
let index__ = 0;
|
||||
index__ < this.special_properties.length;
|
||||
index__++
|
||||
) {
|
||||
const property = this.special_properties[index__];
|
||||
if (property.bone == bone.name + ".quaternion") {
|
||||
const quat_y = THREE.MathUtils.degToRad(property.y);
|
||||
bone.rotateY(quat_y);
|
||||
if (property.only_y == false) {
|
||||
const quat_x = THREE.MathUtils.degToRad(property.x);
|
||||
bone.rotateX(quat_x);
|
||||
const quat_z = THREE.MathUtils.degToRad(property.z);
|
||||
bone.rotateZ(quat_z);
|
||||
} else {
|
||||
if (child_holder.parent) {
|
||||
child_holder.parent.rotation.x =
|
||||
THREE.MathUtils.degToRad(-90);
|
||||
}
|
||||
}
|
||||
|
||||
if (property.flip) {
|
||||
bone.rotateX(bone.rotation.x * 2 * -1);
|
||||
}
|
||||
} else if (property.bone == bone.name + ".position") {
|
||||
bone.position.set(
|
||||
bone.position.x * property.x,
|
||||
bone.position.y * property.y,
|
||||
bone.position.z * property.z,
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async load_vpd_wrapped(): Promise<object> {
|
||||
return new Promise((resolve) => {
|
||||
const poseLoader = new MMDLoader();
|
||||
poseLoader.loadVPD("/resources/pose/7.vpd", false, (vpd) => {
|
||||
resolve(vpd);
|
||||
}); // End of loadVPD call
|
||||
}); // End of new Promise
|
||||
}
|
||||
|
||||
async animate(deltatime: number) {
|
||||
// Sets the frame of the animation mixer.
|
||||
this.mixer?.setTime(deltatime);
|
||||
// Regtargeting for Move.AI
|
||||
this.update_bones();
|
||||
|
||||
// Next
|
||||
if (this.isMMD && this.mixer) {
|
||||
if (this.obj === undefined) {
|
||||
// Sets the ik helper and object for IK and animation.
|
||||
this.helper = (this.mixer.getRoot() as THREE.Object3D).parent?.userData[
|
||||
"helper"
|
||||
];
|
||||
}
|
||||
|
||||
this.obj = this.helper?.objects.get(
|
||||
this.mixer?.getRoot() as THREE.SkinnedMesh,
|
||||
);
|
||||
|
||||
const mesh = this.mixer?.getRoot() as THREE.SkinnedMesh;
|
||||
|
||||
if (this.helper) {
|
||||
this.helper._restoreBones(mesh); // Privateish js function for resetting bones.
|
||||
this.obj?.mixer?.setTime(deltatime); // Sets the time.
|
||||
this.helper._saveBones(mesh); // Saves the bones location for restoration later.
|
||||
|
||||
mesh.updateMatrixWorld(true); // Updates mesh.
|
||||
this.obj?.ikSolver.update(); // Updates IK Solver.
|
||||
this.obj?.grantSolver.update(); // Updates Grant which moves the bones.
|
||||
}
|
||||
|
||||
// Changing of animation.
|
||||
if (
|
||||
this.obj &&
|
||||
this.helper &&
|
||||
this.processingClip === false &&
|
||||
this.obj.mixer &&
|
||||
this.mixer._actions[0]._clip.name !==
|
||||
this.obj.mixer._actions[0]._clip.name &&
|
||||
this.mixer !== undefined
|
||||
) {
|
||||
// On animation change.
|
||||
this.processingClip = true;
|
||||
|
||||
const helper_mmd = this.helper.objects.get(mesh);
|
||||
if (helper_mmd?.mixer !== undefined) {
|
||||
const clips = helper_mmd.mixer._actions;
|
||||
clips.forEach((clip) => {
|
||||
this.helper?.objects.delete(clip._clip);
|
||||
});
|
||||
}
|
||||
|
||||
this.helper.remove(mesh);
|
||||
mesh.pose();
|
||||
|
||||
const loader = new MMDLoader();
|
||||
loader.loadAnimation(this.mmd_url, mesh, (mmd) => {
|
||||
mmd.name = this.mmd_url;
|
||||
this.helper?.add(mesh, {
|
||||
animation: mmd,
|
||||
physics: false,
|
||||
});
|
||||
this.processingClip = false;
|
||||
if (helper_mmd?.mixer !== undefined) {
|
||||
const clips = helper_mmd.mixer._actions;
|
||||
clips.forEach((clip) => {
|
||||
this.helper?.objects.delete(clip._clip);
|
||||
});
|
||||
}
|
||||
this.obj.mixer = this.mixer;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async step(deltatime: number, isPlaying: boolean, frame: number) {
|
||||
if (this.mixer == null) {
|
||||
return;
|
||||
}
|
||||
if (this.retargeted) {
|
||||
if (isPlaying || Math.floor(frame) != this.last_frame) {
|
||||
if (Math.floor(frame) != 0) {
|
||||
await this.animate(deltatime);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await this.animate(deltatime);
|
||||
}
|
||||
this.last_frame = frame;
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.mixer?.setTime(0);
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
version: this.version,
|
||||
media_id: this.media_id,
|
||||
object_uuid: this.object_uuid,
|
||||
type: this.type,
|
||||
speed: this.speed,
|
||||
length: this.length,
|
||||
clip_name: this.clip_name,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { GetCdnOrigin } from "~/api/GetCdnOrigin";
|
||||
import Ijson from "~/interfaces/Ijson";
|
||||
import { StorytellerApiHostStore } from "@storyteller/api";
|
||||
|
||||
interface AudioDataInterface {
|
||||
audioContext: AudioContext;
|
||||
audioBuffer: AudioBuffer;
|
||||
}
|
||||
|
||||
class AudioData implements AudioDataInterface {
|
||||
audioContext: AudioContext;
|
||||
audioBuffer: AudioBuffer;
|
||||
source: AudioBufferSourceNode | undefined;
|
||||
|
||||
constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) {
|
||||
this.audioContext = audioContext;
|
||||
this.audioBuffer = audioBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
export class AudioClip implements Ijson {
|
||||
version: number;
|
||||
media_id: string;
|
||||
type: "audio";
|
||||
volume: number;
|
||||
audio_data: AudioData | undefined;
|
||||
|
||||
constructor(version: number, media_id: string, volume: number) {
|
||||
this.version = version;
|
||||
this.media_id = media_id;
|
||||
this.type = "audio";
|
||||
this.volume = volume;
|
||||
this.download_audio().then((data) => {
|
||||
this.audio_data = data;
|
||||
});
|
||||
}
|
||||
|
||||
async get_media_url() {
|
||||
//This is for prod when we have the proper info on the url.
|
||||
const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`;
|
||||
|
||||
const response = await fetch(url);
|
||||
const json = await JSON.parse(await response.text());
|
||||
const bucketPath = json["media_file"]["public_bucket_path"];
|
||||
const media_api_base_url = GetCdnOrigin();
|
||||
|
||||
const media_url = `${media_api_base_url}${bucketPath}`;
|
||||
return media_url;
|
||||
}
|
||||
|
||||
async download_audio() {
|
||||
const url = await this.get_media_url();
|
||||
const audioContext = new AudioContext();
|
||||
const response = await fetch(url);
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
|
||||
return new AudioData(audioContext, audioBuffer);
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
version: this.version,
|
||||
media_id: this.media_id,
|
||||
type: this.type,
|
||||
volume: this.volume,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
// Clip offsets represent the state of the clip on the timeline as well as what type of clip it is.
|
||||
// it is created from a media id.
|
||||
import { MediaFileType } from "~/pages/PageEnigma/enums";
|
||||
import { ClipGroup, ClipType } from "~/enums";
|
||||
import Ijson from "~/interfaces/Ijson";
|
||||
|
||||
export class ClipUI implements Ijson {
|
||||
version: number;
|
||||
type: ClipType;
|
||||
group: ClipGroup;
|
||||
name: string;
|
||||
media_id: string;
|
||||
object_uuid: string;
|
||||
object_name: string;
|
||||
offset: number; // in frames
|
||||
length: number; // in frames
|
||||
should_play: boolean;
|
||||
clip_uuid: string;
|
||||
keyframe_offset: number;
|
||||
media_file_type: MediaFileType;
|
||||
|
||||
constructor(
|
||||
version: number,
|
||||
type: ClipType,
|
||||
group: ClipGroup,
|
||||
name: string,
|
||||
media_id: string,
|
||||
clip_uuid: string,
|
||||
object_uuid: string,
|
||||
object_name: string,
|
||||
offset: number,
|
||||
length: number,
|
||||
keyframe_offset: number = 0,
|
||||
media_file_type: MediaFileType,
|
||||
) {
|
||||
this.version = version;
|
||||
this.group = group; // Only needed for UI
|
||||
|
||||
this.clip_uuid = clip_uuid;
|
||||
|
||||
this.name = name; // UI
|
||||
this.type = type; // UI and Animation / Audio / Lipsync / : Engine
|
||||
|
||||
this.object_uuid = object_uuid; // Animation / Audio / Lipsync / : Engine
|
||||
this.object_name = object_name;
|
||||
this.media_id = media_id; // Animation / Audio / Lipsync / : Engine
|
||||
this.offset = offset;
|
||||
this.length = length;
|
||||
this.should_play = true;
|
||||
this.keyframe_offset = keyframe_offset;
|
||||
this.media_file_type = media_file_type;
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
version: this.version,
|
||||
group: this.group,
|
||||
name: this.name,
|
||||
type: this.type,
|
||||
clip_uuid: this.clip_uuid,
|
||||
object_uuid: this.object_uuid,
|
||||
object_name: this.object_name,
|
||||
media_id: this.media_id,
|
||||
start_offset: this.offset,
|
||||
ending_offset: this.length,
|
||||
keyframe_offset: this.keyframe_offset,
|
||||
media_file_type: this.media_file_type,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import { GetCdnOrigin } from "~/api/GetCdnOrigin";
|
||||
import Ijson from "~/interfaces/Ijson";
|
||||
import { StorytellerApiHostStore } from "@storyteller/api";
|
||||
|
||||
interface CsvJson {
|
||||
[key: string]: string[];
|
||||
}
|
||||
|
||||
export class EmotionClip implements Ijson {
|
||||
version: number;
|
||||
media_id: string;
|
||||
type: "expression";
|
||||
emotion_json: any;
|
||||
faces: THREE.Mesh[];
|
||||
|
||||
constructor(version: number, media_id: string) {
|
||||
this.version = version;
|
||||
this.media_id = media_id;
|
||||
this.type = "expression";
|
||||
this.faces = [];
|
||||
this.download_csv().then((data) => {
|
||||
this.emotion_json = data;
|
||||
});
|
||||
}
|
||||
|
||||
async get_media_url() {
|
||||
//This is for prod when we have the proper info on the url.
|
||||
const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`;
|
||||
|
||||
const response = await fetch(url);
|
||||
const json = await JSON.parse(await response.text());
|
||||
const bucketPath = json["media_file"]["public_bucket_path"];
|
||||
const media_api_base_url = GetCdnOrigin();
|
||||
const media_url = `${media_api_base_url}${bucketPath}`;
|
||||
return media_url;
|
||||
}
|
||||
|
||||
async download_json() {
|
||||
const url = await this.get_media_url();
|
||||
const response = await fetch(url);
|
||||
return await response.json();
|
||||
}
|
||||
|
||||
async download_csv(): Promise<CsvJson> {
|
||||
const url = await this.get_media_url();
|
||||
const response = await fetch(url);
|
||||
const csvText = await response.text();
|
||||
|
||||
// Parsing the CSV text
|
||||
const rows = csvText.trim().split("\n");
|
||||
const header = rows[0].split(",");
|
||||
const jsonData: CsvJson = {};
|
||||
|
||||
header.forEach((key) => (jsonData[key] = []));
|
||||
|
||||
for (let i = 1; i < rows.length; i++) {
|
||||
const values = rows[i].split(",");
|
||||
values.forEach((value, index) => {
|
||||
jsonData[header[index]].push(value);
|
||||
});
|
||||
}
|
||||
|
||||
return jsonData;
|
||||
}
|
||||
|
||||
async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
|
||||
this.faces = [];
|
||||
return new Promise((resolve) => {
|
||||
object.traverse((c: THREE.Object3D) => {
|
||||
if (c instanceof THREE.Mesh) {
|
||||
if (c.morphTargetInfluences && c.morphTargetDictionary) {
|
||||
const blendShapeIndexE = c.morphTargetDictionary["jawOpen"];
|
||||
if (blendShapeIndexE !== undefined) {
|
||||
this.faces.push(c);
|
||||
resolve(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
setBlends(shapes: { [key: string]: number }) {
|
||||
this.faces.forEach((element: THREE.Mesh) => {
|
||||
if (element.morphTargetInfluences && shapes) {
|
||||
Object.keys(shapes).forEach((key) => {
|
||||
let index = element.morphTargetDictionary?.[key];
|
||||
if (index === undefined) {
|
||||
index =
|
||||
element.morphTargetDictionary?.[
|
||||
key.charAt(0).toLowerCase() + key.slice(1)
|
||||
];
|
||||
}
|
||||
if (
|
||||
typeof index === "number" &&
|
||||
element.morphTargetInfluences !== undefined
|
||||
) {
|
||||
element.morphTargetInfluences[index] = shapes[key];
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async reset(object: THREE.Object3D) {
|
||||
await this._detect_face(object);
|
||||
const keys: { [key: string]: number } = {};
|
||||
if (this.emotion_json === undefined || this.emotion_json === null) {
|
||||
return;
|
||||
}
|
||||
Object.keys(this.emotion_json).forEach((key) => {
|
||||
keys[key] = 0;
|
||||
});
|
||||
this.setBlends(keys);
|
||||
}
|
||||
|
||||
async step(frame: number, object: THREE.Object3D) {
|
||||
if (this.faces.length <= 0) {
|
||||
await this._detect_face(object);
|
||||
}
|
||||
if (this.emotion_json === undefined || this.emotion_json === null) {
|
||||
return;
|
||||
}
|
||||
const keys: { [key: string]: number } = {};
|
||||
Object.keys(this.emotion_json).forEach((key) => {
|
||||
if (Math.floor(frame) > this.emotion_json[key].length) {
|
||||
return;
|
||||
}
|
||||
keys[key] = this.emotion_json[key][Math.floor(frame)];
|
||||
});
|
||||
this.setBlends(keys);
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
version: this.version,
|
||||
media_id: this.media_id,
|
||||
type: this.type,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
import { LipSync } from "../Editor/lipsync";
|
||||
import * as THREE from "three";
|
||||
import { GetCdnOrigin } from "~/api/GetCdnOrigin";
|
||||
import Ijson from "~/interfaces/Ijson";
|
||||
import { StorytellerApiHostStore } from "@storyteller/api";
|
||||
|
||||
interface AudioDataInterface {
|
||||
audioContext: AudioContext;
|
||||
audioBuffer: AudioBuffer;
|
||||
}
|
||||
|
||||
class AudioData implements AudioDataInterface {
|
||||
audioContext: AudioContext;
|
||||
audioBuffer: AudioBuffer;
|
||||
source: AudioBufferSourceNode | undefined;
|
||||
|
||||
constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) {
|
||||
this.audioContext = audioContext;
|
||||
this.audioBuffer = audioBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
class BlendShapeHelper {
|
||||
ah: number;
|
||||
ee: number;
|
||||
oh: number;
|
||||
constructor(ah: number, ee: number, oh: number) {
|
||||
this.ah = ah;
|
||||
this.ee = ee;
|
||||
this.oh = oh;
|
||||
}
|
||||
}
|
||||
|
||||
export class LipSyncClip implements Ijson {
|
||||
version: number;
|
||||
media_id: string;
|
||||
type: "lipsync";
|
||||
volume: number;
|
||||
audio_data: AudioData | undefined;
|
||||
lipsync: LipSync;
|
||||
blendshape_helper: BlendShapeHelper;
|
||||
faces: THREE.Mesh[];
|
||||
|
||||
constructor(version: number, media_id: string, volume: number) {
|
||||
this.version = version;
|
||||
this.media_id = media_id;
|
||||
this.type = "lipsync";
|
||||
this.volume = volume;
|
||||
this.download_audio().then((data) => {
|
||||
this.audio_data = data;
|
||||
});
|
||||
this.blendshape_helper = new BlendShapeHelper(0, 0, 0);
|
||||
// we might need 3 of these one for each character ...
|
||||
this.lipsync = new LipSync();
|
||||
this.faces = [];
|
||||
}
|
||||
|
||||
// lip sync will be generated through TTS
|
||||
async get_media_url() {
|
||||
//This is for prod when we have the proper info on the url.
|
||||
const apiSchemeAndHost = StorytellerApiHostStore.getInstance().getApiSchemeAndHost();
|
||||
const url = `${apiSchemeAndHost}/v1/media_files/file/${this.media_id}`;
|
||||
|
||||
console.log(`API BASE URL? ${apiSchemeAndHost}`);
|
||||
console.log(`CALLED URL? ${url}`);
|
||||
|
||||
const response = await fetch(url);
|
||||
const json = await JSON.parse(await response.text());
|
||||
const bucketPath = json["media_file"]["public_bucket_path"];
|
||||
const media_api_base_url = GetCdnOrigin();
|
||||
//const media_base_url = `${media_api_base_url}/vocodes-public`;
|
||||
//const media_url = `${media_base_url}${bucketPath}`;
|
||||
const media_url = `${media_api_base_url}${bucketPath}`;
|
||||
return media_url;
|
||||
}
|
||||
|
||||
async download_audio() {
|
||||
const url = await this.get_media_url();
|
||||
const audioContext = new AudioContext();
|
||||
const response = await fetch(url);
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
|
||||
return new AudioData(audioContext, audioBuffer);
|
||||
}
|
||||
|
||||
async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
|
||||
return new Promise((resolve) => {
|
||||
object.traverse((c: THREE.Object3D) => {
|
||||
if (c instanceof THREE.Mesh) {
|
||||
if (c.morphTargetInfluences && c.morphTargetDictionary) {
|
||||
let blendShapeIndexE = c.morphTargetDictionary["E"];
|
||||
let blendShapeIndexO = c.morphTargetDictionary["O"];
|
||||
let blendShapeIndexA = c.morphTargetDictionary["aa"];
|
||||
|
||||
if (blendShapeIndexE === undefined) {
|
||||
blendShapeIndexE = c.morphTargetDictionary["お"]; // MMD OH
|
||||
blendShapeIndexO = c.morphTargetDictionary["ω"]; // MMD O
|
||||
blendShapeIndexA = c.morphTargetDictionary["あ"]; // MMD A
|
||||
}
|
||||
|
||||
if (blendShapeIndexE != null) {
|
||||
this.blendshape_helper = new BlendShapeHelper(
|
||||
blendShapeIndexA,
|
||||
blendShapeIndexE,
|
||||
blendShapeIndexO,
|
||||
);
|
||||
this.faces.push(c);
|
||||
resolve(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async play(object: THREE.Object3D) {
|
||||
if (this.audio_data?.audioBuffer == null) {
|
||||
await this.download_audio();
|
||||
}
|
||||
if (this.lipsync.face == null) {
|
||||
this.lipsync = new LipSync(await this._detect_face(object));
|
||||
this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (this.lipsync == null) {
|
||||
return;
|
||||
}
|
||||
this.lipsync.destroy();
|
||||
}
|
||||
|
||||
setBlends(ah: number, ee: number, oh: number) {
|
||||
this.faces.forEach((element: THREE.Mesh) => {
|
||||
if (element.morphTargetInfluences) {
|
||||
element.morphTargetInfluences[this.blendshape_helper.ee] = ee;
|
||||
element.morphTargetInfluences[this.blendshape_helper.ah] = ah;
|
||||
element.morphTargetInfluences[this.blendshape_helper.oh] = oh;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
step(
|
||||
frame: number,
|
||||
offset: number,
|
||||
// rendering: boolean
|
||||
) {
|
||||
if (this.lipsync == null) {
|
||||
return;
|
||||
}
|
||||
const positions = this.lipsync.update(frame, offset, true);
|
||||
if (positions)
|
||||
this.setBlends(positions["ee"], positions["ah"], positions["oh"]);
|
||||
}
|
||||
|
||||
reset() {
|
||||
if (this.lipsync.face != undefined) {
|
||||
this.setBlends(0, 0, 0);
|
||||
}
|
||||
this.lipsync = new LipSync();
|
||||
}
|
||||
|
||||
toJSON() {
|
||||
return {
|
||||
version: this.version,
|
||||
media_id: this.media_id,
|
||||
type: this.type,
|
||||
volume: this.volume,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
import * as THREE from "three";
|
||||
import Scene from "../Editor/scene";
|
||||
|
||||
export class TransformFrame {
|
||||
position: THREE.Vector3;
|
||||
rotation: THREE.Vector3;
|
||||
scale: THREE.Vector3;
|
||||
offset: number;
|
||||
keyframe_uuid: string;
|
||||
|
||||
constructor(
|
||||
position: THREE.Vector3,
|
||||
rotation: THREE.Vector3,
|
||||
scale: THREE.Vector3,
|
||||
offset: number,
|
||||
keyframe_uuid: string,
|
||||
) {
|
||||
this.position = position;
|
||||
this.rotation = rotation;
|
||||
this.scale = scale;
|
||||
this.offset = offset;
|
||||
this.keyframe_uuid = keyframe_uuid;
|
||||
}
|
||||
}
|
||||
|
||||
export class TransformClip {
|
||||
version: number;
|
||||
media_id: string;
|
||||
object_uuid: string;
|
||||
type: "transform";
|
||||
keyframes: TransformFrame[];
|
||||
length: number;
|
||||
|
||||
step_frame: number;
|
||||
looping: boolean;
|
||||
|
||||
offset: number;
|
||||
|
||||
constructor(
|
||||
version: number,
|
||||
object_uuid: string,
|
||||
length: number,
|
||||
media_id: string = "",
|
||||
) {
|
||||
this.version = version;
|
||||
this.media_id = media_id;
|
||||
this.object_uuid = object_uuid;
|
||||
this.type = "transform";
|
||||
|
||||
this.length = length;
|
||||
|
||||
this.keyframes = [];
|
||||
|
||||
this.step_frame = 0;
|
||||
this.offset = 0;
|
||||
|
||||
this.looping = false;
|
||||
}
|
||||
|
||||
private findNextNumber(location: number) {
|
||||
// Sort the transform frames based on their offset in ascending order
|
||||
this.keyframes.sort((a, b) => a.offset - b.offset);
|
||||
|
||||
let currentKeyframe: TransformFrame | undefined;
|
||||
let nextKeyframe: TransformFrame | undefined;
|
||||
|
||||
this.length =
|
||||
this.keyframes[this.keyframes.length - 1].offset + this.offset;
|
||||
|
||||
for (const frame of this.keyframes) {
|
||||
if (frame.offset <= location) {
|
||||
currentKeyframe = frame;
|
||||
} else if (frame.offset > location) {
|
||||
nextKeyframe = frame;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { currentKeyframe, nextKeyframe };
|
||||
}
|
||||
|
||||
public setOffset(keyframe_uuid: string, keyframe_offset: number) {
|
||||
this.keyframes.forEach((item) => {
|
||||
if (item.keyframe_uuid === keyframe_uuid) {
|
||||
item.offset = keyframe_offset;
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public setTransform(
|
||||
keyframe_uuid: string,
|
||||
position: THREE.Vector3,
|
||||
rotation: THREE.Vector3,
|
||||
scale: THREE.Vector3,
|
||||
) {
|
||||
this.keyframes.forEach((item) => {
|
||||
if (item.keyframe_uuid === keyframe_uuid) {
|
||||
item.position = position;
|
||||
item.rotation = rotation;
|
||||
item.scale = scale;
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public removeKeyframe(keyframe_uuid: string) {
|
||||
this.keyframes = this.keyframes.filter(
|
||||
(item) => item.keyframe_uuid !== keyframe_uuid,
|
||||
);
|
||||
}
|
||||
|
||||
request_point_location(keyframe_uuid: string, scene: Scene) {
|
||||
const point = scene.getPoint(keyframe_uuid);
|
||||
if (point == undefined) {
|
||||
console.log("Could not find point.");
|
||||
return;
|
||||
}
|
||||
this.keyframes.forEach((keyframe) => {
|
||||
if (keyframe.keyframe_uuid == keyframe_uuid) {
|
||||
keyframe.position = point.position;
|
||||
keyframe.rotation = new THREE.Vector3(
|
||||
point.rotation.x,
|
||||
point.rotation.y,
|
||||
point.rotation.z,
|
||||
);
|
||||
keyframe.scale = point.scale;
|
||||
}
|
||||
});
|
||||
return point;
|
||||
}
|
||||
|
||||
step(object: THREE.Object3D, offset: number, frame: number, scene: Scene) {
|
||||
this.offset = offset;
|
||||
this.step_frame = frame - offset;
|
||||
//if (this.step_frame < 0) { this.step_frame = 0; }
|
||||
//if (this.step_frame >= this.length) { return; } // Reached max frames.
|
||||
if (this.keyframes.length < 1) {
|
||||
return;
|
||||
} // If there are enough points in the scene.
|
||||
|
||||
// Find the current and next keyframes based on time_frame
|
||||
const { currentKeyframe, nextKeyframe } = this.findNextNumber(
|
||||
this.step_frame,
|
||||
);
|
||||
if (nextKeyframe != undefined && currentKeyframe != undefined) {
|
||||
const small_step_frame = this.step_frame - currentKeyframe.offset;
|
||||
let location =
|
||||
small_step_frame / (nextKeyframe.offset - currentKeyframe.offset);
|
||||
if (location < 0) {
|
||||
location = 0;
|
||||
}
|
||||
const pos_s = currentKeyframe?.position;
|
||||
let pos_e = nextKeyframe?.position;
|
||||
if (pos_e === undefined) {
|
||||
this.request_point_location(nextKeyframe.keyframe_uuid, scene);
|
||||
pos_e = nextKeyframe?.position;
|
||||
}
|
||||
const points_pos = [
|
||||
new THREE.Vector3(pos_s.x, pos_s.y, pos_s.z),
|
||||
new THREE.Vector3(pos_e.x, pos_e.y, pos_e.z),
|
||||
];
|
||||
const curve_pos = new THREE.CatmullRomCurve3(points_pos);
|
||||
const point_pos = curve_pos.getPoint(location);
|
||||
object.position.copy(point_pos);
|
||||
|
||||
const rot_s = currentKeyframe?.rotation;
|
||||
const rot_e = nextKeyframe?.rotation;
|
||||
const quat_s = new THREE.Quaternion().setFromEuler(
|
||||
new THREE.Euler(
|
||||
THREE.MathUtils.degToRad(rot_s.x),
|
||||
THREE.MathUtils.degToRad(rot_s.y),
|
||||
THREE.MathUtils.degToRad(rot_s.z),
|
||||
),
|
||||
);
|
||||
const quat_e = new THREE.Quaternion().setFromEuler(
|
||||
new THREE.Euler(
|
||||
THREE.MathUtils.degToRad(rot_e.x),
|
||||
THREE.MathUtils.degToRad(rot_e.y),
|
||||
THREE.MathUtils.degToRad(rot_e.z),
|
||||
),
|
||||
);
|
||||
const quaternion = new THREE.Quaternion();
|
||||
quaternion.slerpQuaternions(quat_s, quat_e, location);
|
||||
object.quaternion.copy(quaternion);
|
||||
|
||||
const scale_s = currentKeyframe?.scale;
|
||||
const scale_e = nextKeyframe?.scale;
|
||||
const points_scale = [
|
||||
new THREE.Vector3(scale_s.x, scale_s.y, scale_s.z),
|
||||
new THREE.Vector3(scale_e.x, scale_e.y, scale_e.z),
|
||||
];
|
||||
const curve_scale = new THREE.CatmullRomCurve3(points_scale);
|
||||
const point_scale = curve_scale.getPoint(location);
|
||||
object.scale.copy(point_scale);
|
||||
}
|
||||
}
|
||||
|
||||
reset(object: THREE.Object3D) {
|
||||
if (this.keyframes.length > 0) {
|
||||
const first_pos = this.keyframes[0].position;
|
||||
const first_rot = this.keyframes[0].rotation;
|
||||
const first_quat = new THREE.Euler(
|
||||
THREE.MathUtils.degToRad(first_rot.x),
|
||||
THREE.MathUtils.degToRad(first_rot.y),
|
||||
THREE.MathUtils.degToRad(first_rot.z),
|
||||
);
|
||||
const first_scl = this.keyframes[0].scale;
|
||||
object.position.copy(first_pos);
|
||||
object.rotation.copy(first_quat);
|
||||
object.scale.copy(first_scl);
|
||||
this.step_frame = 0;
|
||||
}
|
||||
}
|
||||
|
||||
add_frame(
|
||||
position: THREE.Vector3,
|
||||
rotation: THREE.Vector3,
|
||||
scale: THREE.Vector3,
|
||||
offset: number,
|
||||
keyframe_uuid: string,
|
||||
) {
|
||||
this.keyframes.push(
|
||||
new TransformFrame(position, rotation, scale, offset, keyframe_uuid),
|
||||
);
|
||||
}
|
||||
|
||||
toJSON(): any {
|
||||
return {
|
||||
version: this.version,
|
||||
media_id: this.media_id,
|
||||
object_uuid: this.object_uuid,
|
||||
length: this.length,
|
||||
type: this.type,
|
||||
keyframes: this.keyframes,
|
||||
};
|
||||
}
|
||||
}
|
||||
Whitespace-only changes.
Whitespace-only changes.
-158
@@ -1,158 +0,0 @@
|
||||
import { useContext, useState } from "react";
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
import {
|
||||
faBackwardStep,
|
||||
faForwardStep,
|
||||
faCirclePause,
|
||||
faCirclePlay,
|
||||
faPlay,
|
||||
faRepeat,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { ButtonIcon } from "@storyteller/ui-button-icon";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
|
||||
import {
|
||||
currentTime,
|
||||
filmLength,
|
||||
secondaryScrubber,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
import { editorState } from "~/pages/PageEnigma/signals/engine";
|
||||
import { EditorStates } from "~/pages/PageEnigma/enums";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
import { usePosthogFeatureFlag } from "~/hooks/usePosthogFeatureFlag";
|
||||
import { FeatureFlags } from "~/enums";
|
||||
|
||||
export const ControlsVideo = () => {
|
||||
useSignals();
|
||||
const editorEngine = useContext(EngineContext);
|
||||
const [refresh, setRefresh] = useState(0);
|
||||
const showStylePage = usePosthogFeatureFlag(FeatureFlags.SHOW_STYLE_PAGE);
|
||||
|
||||
function formatTime(millis: number) {
|
||||
const date = new Date(millis);
|
||||
const secs = String(date.getUTCSeconds()).padStart(2, "0");
|
||||
const ms = String(date.getUTCMilliseconds()).padStart(3, "0");
|
||||
return `00:${secs}:${ms}`;
|
||||
}
|
||||
|
||||
const isPlaying =
|
||||
editorEngine !== null ? editorEngine.timeline.is_playing : false;
|
||||
const isRepeating =
|
||||
editorEngine !== null ? editorEngine.timeline.is_repeating : false;
|
||||
|
||||
const handleToStart = () => {
|
||||
currentTime.value = 0;
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_TIME,
|
||||
data: { currentTime: currentTime.value },
|
||||
});
|
||||
};
|
||||
|
||||
const handleToEnd = () => {
|
||||
currentTime.value = filmLength.value * 1000;
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_TIME,
|
||||
data: { currentTime: currentTime.value },
|
||||
});
|
||||
};
|
||||
|
||||
const handleBackwardStep = () => {
|
||||
currentTime.value = Math.max(currentTime.value - 1, 0);
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_TIME,
|
||||
data: { currentTime: currentTime.value },
|
||||
});
|
||||
};
|
||||
const handleRepeat = () => {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.TOGGLE_REPEATING,
|
||||
data: null,
|
||||
});
|
||||
setRefresh((refresh + 1) % 10);
|
||||
};
|
||||
const handlePlayback = () => {
|
||||
editorEngine?.togglePlayback();
|
||||
};
|
||||
const handleForwardStep = () => {
|
||||
currentTime.value = Math.min(
|
||||
currentTime.value + 1,
|
||||
filmLength.value * 1000,
|
||||
);
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_TIME,
|
||||
data: { currentTime: currentTime.value },
|
||||
});
|
||||
};
|
||||
|
||||
if (editorState.value === EditorStates.PREVIEW) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-center">
|
||||
<div className="flex items-center gap-4 rounded-t-lg bg-ui-controls p-1.5 text-white shadow-md">
|
||||
<div className="flex items-center gap-1">
|
||||
<div className="flex gap-0">
|
||||
<ButtonIcon icon={faBackwardStep} onClick={handleToStart} />
|
||||
<ButtonIcon
|
||||
icon={faPlay}
|
||||
onClick={handleBackwardStep}
|
||||
className="rotate-180 text-sm"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<ButtonIcon
|
||||
icon={isPlaying ? faCirclePause : faCirclePlay}
|
||||
onClick={handlePlayback}
|
||||
className="p-0 text-2xl"
|
||||
/>
|
||||
|
||||
<div className="flex gap-0">
|
||||
<ButtonIcon
|
||||
icon={faPlay}
|
||||
onClick={handleForwardStep}
|
||||
className="text-sm"
|
||||
/>
|
||||
<ButtonIcon icon={faForwardStep} onClick={handleToEnd} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="mr-2 flex items-center gap-1.5 text-sm font-medium">
|
||||
<span className="w-[54px]">
|
||||
{formatTime(secondaryScrubber.value)}
|
||||
</span>
|
||||
<span className="opacity-50">/</span>
|
||||
<span className="w-[54px] opacity-50">
|
||||
{formatTime(filmLength.value * 1000)}
|
||||
</span>
|
||||
</div>
|
||||
{showStylePage && (
|
||||
<>
|
||||
<div className="h-[18px] w-0.5 rounded-full bg-white/20" />
|
||||
<Tooltip content="Loop" position={"top"}>
|
||||
<ButtonIcon
|
||||
icon={faRepeat}
|
||||
onClick={handleRepeat}
|
||||
className={twMerge(
|
||||
"h-7 w-7 p-0 text-sm",
|
||||
isRepeating
|
||||
? "border border-brand-primary"
|
||||
: "border border-transparent",
|
||||
)}
|
||||
/>
|
||||
</Tooltip>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1 +0,0 @@
|
||||
export { ControlsVideo } from './ControlsVideo';
|
||||
@@ -1,84 +0,0 @@
|
||||
import {
|
||||
filmLength,
|
||||
frameTrackButtonWidthPx,
|
||||
fullHeight,
|
||||
scale,
|
||||
timelineScrollX,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import { Fragment } from "react";
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
|
||||
export const TimerGrid = () => {
|
||||
useSignals();
|
||||
const sectionWidth = 1000 * 4 * scale.value;
|
||||
const scrollX = timelineScrollX.value;
|
||||
|
||||
return (
|
||||
<div
|
||||
className={[
|
||||
"prevent-select mt-2",
|
||||
"relative flex h-5 overflow-hidden",
|
||||
"border-t border-t-ui-panel-border",
|
||||
"text-xs text-white opacity-75",
|
||||
].join(" ")}
|
||||
|
||||
style={{
|
||||
marginLeft: `${204 + frameTrackButtonWidthPx}px`
|
||||
}}
|
||||
>
|
||||
<div className="absolute" style={{ left: scrollX * -1 }}>
|
||||
{Array(filmLength.value)
|
||||
.fill(0)
|
||||
.map((_, index) => (
|
||||
<Fragment key={index}>
|
||||
<div
|
||||
className="absolute ps-0.5 pt-1.5 text-[11px]"
|
||||
style={{ left: index * sectionWidth + 4 }}
|
||||
>
|
||||
00:{index < 10 ? "0" + index.toString() : index.toString()}
|
||||
</div>
|
||||
<div
|
||||
className="absolute mt-0.5 block bg-ui-divider"
|
||||
style={{
|
||||
width: 1,
|
||||
left: index * sectionWidth,
|
||||
height: fullHeight.value,
|
||||
}}
|
||||
/>
|
||||
{Array(9)
|
||||
.fill(0)
|
||||
.map((_, ind) => (
|
||||
<div
|
||||
key={ind}
|
||||
className="absolute mt-0.5 block h-1 bg-ui-divider"
|
||||
style={{
|
||||
width: 1,
|
||||
top: 0,
|
||||
left:
|
||||
index * sectionWidth + (sectionWidth / 10) * (ind + 1),
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</Fragment>
|
||||
))}
|
||||
<div
|
||||
className="absolute ps-0.5 pt-1.5 text-[11px]"
|
||||
style={{ left: filmLength.value * sectionWidth + 4 }}
|
||||
>
|
||||
00:
|
||||
{filmLength.value < 10
|
||||
? "0" + filmLength.value.toString()
|
||||
: filmLength.value.toString()}
|
||||
</div>
|
||||
<div
|
||||
className="absolute block h-full bg-ui-divider"
|
||||
style={{
|
||||
width: 1,
|
||||
left: filmLength.value * sectionWidth,
|
||||
height: fullHeight.value,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div >
|
||||
);
|
||||
};
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
import { createContext } from "react";
|
||||
import { MediaItem } from "~/pages/PageEnigma/models";
|
||||
|
||||
export const TrackContext = createContext<{
|
||||
// drag and drop
|
||||
startDrag: (item: MediaItem) => void;
|
||||
endDrag: () => void;
|
||||
}>({
|
||||
startDrag: () => {},
|
||||
endDrag: () => {},
|
||||
});
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
|
||||
import { ReactNode, useCallback, useMemo } from "react";
|
||||
import useUpdateDragDrop from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop";
|
||||
|
||||
interface Props {
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
export const TrackProvider = ({ children }: Props) => {
|
||||
const { endDrag, ...dragDrop } = useUpdateDragDrop();
|
||||
|
||||
// cross group functions
|
||||
const dropClip = useCallback(() => {
|
||||
endDrag();
|
||||
}, [endDrag]);
|
||||
|
||||
const values = useMemo(() => {
|
||||
return {
|
||||
...dragDrop,
|
||||
endDrag: dropClip,
|
||||
};
|
||||
}, [dragDrop, dropClip]);
|
||||
|
||||
return (
|
||||
<TrackContext.Provider value={values}>{children}</TrackContext.Provider>
|
||||
);
|
||||
};
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
import { useCallback } from "react";
|
||||
|
||||
export default function useUpdateDragDrop() {
|
||||
const startDrag = useCallback(() => {}, []);
|
||||
|
||||
const endDrag = useCallback(() => {}, []);
|
||||
|
||||
return {
|
||||
startDrag,
|
||||
endDrag,
|
||||
};
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
import { useCallback, useEffect } from "react";
|
||||
import {
|
||||
addNewCharacter,
|
||||
addObjectToTimeline,
|
||||
clearExistingData,
|
||||
currentTime,
|
||||
deleteObjectOrCharacter,
|
||||
loadAudioData,
|
||||
loadCameraData,
|
||||
loadCharacterData,
|
||||
loadObjectData,
|
||||
selectedObject,
|
||||
addKeyframe,
|
||||
selectedItem,
|
||||
generateProgress,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import Queue, { QueueSubscribeType } from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
|
||||
import { ToastDataType } from "~/components";
|
||||
|
||||
import {
|
||||
MediaItem,
|
||||
QueueKeyframe,
|
||||
UpdateTime,
|
||||
} from "~/pages/PageEnigma/models";
|
||||
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
|
||||
import { addToast } from "~/signals";
|
||||
import { ToastTypes, ClipGroup } from "~/enums";
|
||||
import { toTimelineActions } from "~/pages/PageEnigma/Queue/toTimelineActions";
|
||||
import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui";
|
||||
import { cameraAspectRatio } from "../signals/engine";
|
||||
import { loadPromptTravelData } from "~/pages/PageEnigma/signals/promptTravelGroup";
|
||||
|
||||
const LOADING_FUNCTIONS: Record<ClipGroup, (item: ClipUI) => void> = {
|
||||
[ClipGroup.CHARACTER]: loadCharacterData,
|
||||
[ClipGroup.OBJECT]: loadObjectData,
|
||||
[ClipGroup.CAMERA]: loadCameraData,
|
||||
[ClipGroup.GLOBAL_AUDIO]: loadAudioData,
|
||||
[ClipGroup.PROMPT_TRAVEL]: loadPromptTravelData,
|
||||
};
|
||||
|
||||
export function useQueueHandler() {
|
||||
useSignals();
|
||||
|
||||
const handleFromEngineActions = useCallback(
|
||||
({ action, data }: QueueSubscribeType) => {
|
||||
//console.log("FROM ENGINE", action, data);
|
||||
switch (action) {
|
||||
case fromEngineActions.ADD_OBJECT: {
|
||||
// this could be an object or character
|
||||
addObjectToTimeline(data as MediaItem);
|
||||
break;
|
||||
}
|
||||
case fromEngineActions.DELETE_OBJECT:
|
||||
// this could be an object or character
|
||||
deleteObjectOrCharacter(data as MediaItem);
|
||||
break;
|
||||
case fromEngineActions.DESELECT_OBJECT:
|
||||
selectedObject.value = null;
|
||||
break;
|
||||
case fromEngineActions.RESET_TIMELINE:
|
||||
clearExistingData();
|
||||
break;
|
||||
case fromEngineActions.SELECT_OBJECT:
|
||||
selectedObject.value = {
|
||||
type: (data as MediaItem).type,
|
||||
id: (data as MediaItem).object_uuid ?? "",
|
||||
};
|
||||
// Deselect clip item when scene item is selected
|
||||
selectedItem.value = null;
|
||||
break;
|
||||
case fromEngineActions.UPDATE_CHARACTER_ID:
|
||||
addNewCharacter(data as MediaItem);
|
||||
break;
|
||||
case fromEngineActions.UPDATE_TIME:
|
||||
currentTime.value = (data as UpdateTime).currentTime;
|
||||
break;
|
||||
case fromEngineActions.UPDATE_TIME_LINE:
|
||||
clearExistingData();
|
||||
(data as ClipUI[]).forEach((item) => {
|
||||
LOADING_FUNCTIONS[item.group](item);
|
||||
});
|
||||
break;
|
||||
case fromEngineActions.POP_A_TOAST: {
|
||||
const message = (data as ToastDataType).message;
|
||||
addToast(ToastTypes.ERROR, message);
|
||||
break;
|
||||
}
|
||||
case fromEngineActions.CAMERA_ASPECT_RATIO_CHANGED: {
|
||||
cameraAspectRatio.value = data as CameraAspectRatio;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown action ${action}`);
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const handleToTimelineActions = useCallback(
|
||||
({ action, data }: QueueSubscribeType) => {
|
||||
//console.log("TO TIMELINE", action, data);
|
||||
switch (action) {
|
||||
case toTimelineActions.ADD_KEYFRAME:
|
||||
addKeyframe(data as QueueKeyframe, currentTime.value);
|
||||
break;
|
||||
case toTimelineActions.GENERATE_PROGRESS:
|
||||
generateProgress.value = (
|
||||
data as { currentTime: number }
|
||||
).currentTime;
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown action ${action}`);
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
Queue.subscribe(QueueNames.FROM_ENGINE, "useQ", handleFromEngineActions);
|
||||
Queue.subscribe(QueueNames.TO_TIMELINE, "useQ", handleToTimelineActions);
|
||||
}, [handleFromEngineActions, handleToTimelineActions]);
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
import { AudioGroup, Clip, MediaItem } from "~/pages/PageEnigma/models";
|
||||
import { ClipGroup, ClipType } from "~/enums";
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
import * as uuid from "uuid";
|
||||
import { signal } from "@preact/signals-core";
|
||||
import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui";
|
||||
import { publishClip } from "~/pages/PageEnigma/signals/utils/publishClip";
|
||||
|
||||
export const audioGroup = signal<AudioGroup>({
|
||||
id: "AG-1",
|
||||
clips: [],
|
||||
muted: false,
|
||||
});
|
||||
|
||||
export function updateAudio({
|
||||
id,
|
||||
offset,
|
||||
length,
|
||||
}: {
|
||||
id: string;
|
||||
length: number;
|
||||
offset: number;
|
||||
}) {
|
||||
const oldAudioGroup = audioGroup.value;
|
||||
const newClips = [...oldAudioGroup.clips];
|
||||
const clipIndex = newClips.findIndex((row) => row.clip_uuid === id);
|
||||
if (clipIndex === -1) {
|
||||
return { ...oldAudioGroup };
|
||||
}
|
||||
const clip = newClips[clipIndex];
|
||||
clip.offset = offset;
|
||||
clip.length = length;
|
||||
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_CLIP,
|
||||
data: clip,
|
||||
});
|
||||
|
||||
audioGroup.value = {
|
||||
...oldAudioGroup,
|
||||
clips: newClips,
|
||||
};
|
||||
}
|
||||
|
||||
export function addGlobalAudio({
|
||||
dragItem,
|
||||
audioId,
|
||||
offset,
|
||||
}: {
|
||||
dragItem: MediaItem;
|
||||
audioId: string;
|
||||
offset: number;
|
||||
}) {
|
||||
if (audioGroup.value.id !== audioId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const clip_uuid = uuid.v4();
|
||||
const newClip = {
|
||||
version: 1,
|
||||
media_id: dragItem.media_id,
|
||||
type: ClipType.AUDIO,
|
||||
group: ClipGroup.GLOBAL_AUDIO,
|
||||
offset,
|
||||
length: dragItem.length,
|
||||
name: dragItem.name,
|
||||
clip_uuid,
|
||||
} as Clip;
|
||||
|
||||
publishClip(newClip, dragItem, offset);
|
||||
|
||||
const oldAudioGroup = audioGroup.value;
|
||||
audioGroup.value = {
|
||||
...oldAudioGroup,
|
||||
clips: [...oldAudioGroup.clips, newClip],
|
||||
};
|
||||
}
|
||||
|
||||
export function toggleAudioMute() {
|
||||
const oldAudioGroup = audioGroup.value;
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: oldAudioGroup?.muted
|
||||
? toEngineActions.UNMUTE
|
||||
: toEngineActions.MUTE,
|
||||
data: {
|
||||
version: 1,
|
||||
type: ClipType.AUDIO,
|
||||
group: ClipGroup.GLOBAL_AUDIO,
|
||||
},
|
||||
});
|
||||
|
||||
audioGroup.value = {
|
||||
...oldAudioGroup,
|
||||
muted: !oldAudioGroup.muted,
|
||||
};
|
||||
}
|
||||
|
||||
export function selectAudioClip(clipId: string) {
|
||||
const oldAudioGroup = audioGroup.value;
|
||||
audioGroup.value = {
|
||||
...oldAudioGroup,
|
||||
clips: [
|
||||
...oldAudioGroup.clips.map((clip) => {
|
||||
return {
|
||||
...clip,
|
||||
selected: clip.clip_uuid === clipId ? !clip.selected : clip.selected,
|
||||
};
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function deleteAudioClip(clip: Clip) {
|
||||
const oldAudioGroup = audioGroup.value;
|
||||
audioGroup.value = {
|
||||
...oldAudioGroup,
|
||||
clips: [
|
||||
...oldAudioGroup.clips.filter((row) => {
|
||||
if (row.clip_uuid === clip.clip_uuid) {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.DELETE_CLIP,
|
||||
data: row,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function loadAudioData(item: ClipUI) {
|
||||
const existingAudio = audioGroup.value;
|
||||
const newItem = {
|
||||
version: item.version,
|
||||
clip_uuid: item.clip_uuid,
|
||||
type: item.type,
|
||||
group: item.group,
|
||||
object_uuid: item.object_uuid,
|
||||
media_id: item.media_id,
|
||||
name: item.name,
|
||||
offset: item.offset,
|
||||
length: item.length,
|
||||
} as Clip;
|
||||
existingAudio.clips.push(newItem);
|
||||
existingAudio.clips.sort((clipA, clipB) => clipA.offset - clipB.offset);
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
import {
|
||||
CameraGroup,
|
||||
Keyframe,
|
||||
QueueKeyframe,
|
||||
} from "~/pages/PageEnigma/models";
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
import * as uuid from "uuid";
|
||||
import { signal } from "@preact/signals-core";
|
||||
import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui";
|
||||
import { ToastTypes } from "~/enums";
|
||||
import { addToast } from "~/signals";
|
||||
|
||||
export const cameraGroup = signal<CameraGroup>({ id: "CG1", keyframes: [] });
|
||||
|
||||
export function updateCamera({ id, offset }: { id: string; offset: number }) {
|
||||
const oldCameraGroup = cameraGroup.value;
|
||||
|
||||
const existingKeyframe = oldCameraGroup.keyframes.find((row) => {
|
||||
return row.offset === offset && row.keyframe_uuid !== id;
|
||||
});
|
||||
|
||||
if (existingKeyframe) {
|
||||
addToast(
|
||||
ToastTypes.WARNING,
|
||||
"There can only be one keyframe at this offset.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const newKeyframes = [...oldCameraGroup.keyframes];
|
||||
const keyframe = newKeyframes.find((row) => row.keyframe_uuid === id);
|
||||
if (!keyframe) {
|
||||
return;
|
||||
}
|
||||
keyframe.offset = offset;
|
||||
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_KEYFRAME,
|
||||
data: keyframe,
|
||||
});
|
||||
|
||||
cameraGroup.value = {
|
||||
...oldCameraGroup,
|
||||
keyframes: newKeyframes,
|
||||
};
|
||||
}
|
||||
|
||||
export function addCameraKeyframe(keyframe: QueueKeyframe, offset: number) {
|
||||
const oldCameraGroup = cameraGroup.value;
|
||||
|
||||
const newKeyframe = {
|
||||
version: keyframe.version,
|
||||
keyframe_uuid: uuid.v4(),
|
||||
group: keyframe.group,
|
||||
object_uuid: keyframe.object_uuid,
|
||||
offset,
|
||||
position: keyframe.position,
|
||||
rotation: keyframe.rotation,
|
||||
scale: keyframe.scale,
|
||||
selected: false,
|
||||
} as Keyframe;
|
||||
|
||||
const existingKeyframe = oldCameraGroup.keyframes.find(
|
||||
(row) => row.offset === offset,
|
||||
);
|
||||
if (existingKeyframe) {
|
||||
addToast(
|
||||
ToastTypes.WARNING,
|
||||
"There can only be one keyframe at this offset.",
|
||||
);
|
||||
newKeyframe.keyframe_uuid = existingKeyframe.keyframe_uuid;
|
||||
}
|
||||
|
||||
cameraGroup.value = {
|
||||
...oldCameraGroup,
|
||||
keyframes: [
|
||||
...oldCameraGroup.keyframes.filter(
|
||||
(keyframe) =>
|
||||
keyframe.keyframe_uuid !== existingKeyframe?.keyframe_uuid,
|
||||
),
|
||||
newKeyframe,
|
||||
].sort((keyFrameA, keyframeB) => keyFrameA.offset - keyframeB.offset),
|
||||
};
|
||||
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: existingKeyframe
|
||||
? toEngineActions.UPDATE_KEYFRAME
|
||||
: toEngineActions.ADD_KEYFRAME,
|
||||
data: newKeyframe,
|
||||
});
|
||||
}
|
||||
|
||||
export function selectCameraKeyframe(keyframeId: string) {
|
||||
const oldCameraGroup = cameraGroup.value;
|
||||
cameraGroup.value = {
|
||||
...oldCameraGroup,
|
||||
keyframes: [
|
||||
...oldCameraGroup.keyframes.map((keyframe) => {
|
||||
return {
|
||||
...keyframe,
|
||||
selected:
|
||||
keyframe.keyframe_uuid === keyframeId
|
||||
? !keyframe.selected
|
||||
: keyframe.selected,
|
||||
};
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function deleteCameraKeyframe(deleteKeyframe: Keyframe) {
|
||||
const oldCameraGroup = cameraGroup.value;
|
||||
cameraGroup.value = {
|
||||
...oldCameraGroup,
|
||||
keyframes: [
|
||||
...oldCameraGroup.keyframes.filter((keyframe) => {
|
||||
if (keyframe.keyframe_uuid === deleteKeyframe.keyframe_uuid) {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.DELETE_KEYFRAME,
|
||||
data: keyframe,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function loadCameraData(item: ClipUI) {
|
||||
const existingCamera = cameraGroup.value;
|
||||
const newKeyframe = {
|
||||
version: item.version,
|
||||
keyframe_uuid: item.clip_uuid,
|
||||
group: item.group,
|
||||
object_uuid: item.object_uuid,
|
||||
offset: item.keyframe_offset,
|
||||
} as Keyframe;
|
||||
existingCamera.keyframes.push(newKeyframe);
|
||||
existingCamera.keyframes.sort(
|
||||
(keyframeA, keyframeB) => keyframeA.offset - keyframeB.offset,
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
export * from "./audioGroup";
|
||||
export * from "./cameraGroup";
|
||||
export * from "./characterGroups";
|
||||
export * from "./demoAssets";
|
||||
export * from "./editorOverlays";
|
||||
@@ -15,7 +13,5 @@ export * from "./shape";
|
||||
export * from "./sidePanel";
|
||||
export * from "./sizing";
|
||||
export * from "./stylizeTab";
|
||||
export * from "./timing";
|
||||
export * from "./timeline";
|
||||
export * from "./error";
|
||||
export * from "./assetModal";
|
||||
@@ -1,120 +0,0 @@
|
||||
import { Clip, MediaItem, PromptTravelGroup } from "~/pages/PageEnigma/models";
|
||||
import { ClipGroup, ClipType } from "~/enums";
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
import * as uuid from "uuid";
|
||||
import { signal } from "@preact/signals-core";
|
||||
import { ClipUI } from "~/pages/PageEnigma/clips/clip_ui";
|
||||
import { publishClip } from "~/pages/PageEnigma/signals/utils/publishClip";
|
||||
|
||||
export const promptTravelGroup = signal<PromptTravelGroup>({
|
||||
id: "AG-1",
|
||||
clips: [],
|
||||
});
|
||||
|
||||
export function updatePromptTravel({
|
||||
id,
|
||||
offset,
|
||||
length,
|
||||
name,
|
||||
}: {
|
||||
id: string;
|
||||
length: number;
|
||||
offset: number;
|
||||
name: string;
|
||||
}) {
|
||||
const oldPromptTravelGroup = promptTravelGroup.value;
|
||||
const newClips = [...oldPromptTravelGroup.clips];
|
||||
const clipIndex = newClips.findIndex((row) => row.clip_uuid === id);
|
||||
if (clipIndex === -1) {
|
||||
return { ...oldPromptTravelGroup };
|
||||
}
|
||||
const clip = newClips[clipIndex];
|
||||
clip.offset = offset;
|
||||
clip.length = length;
|
||||
clip.name = name;
|
||||
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_CLIP,
|
||||
data: clip,
|
||||
});
|
||||
|
||||
promptTravelGroup.value = {
|
||||
...oldPromptTravelGroup,
|
||||
clips: newClips,
|
||||
};
|
||||
}
|
||||
|
||||
export function addPromptTravel({
|
||||
text,
|
||||
length,
|
||||
offset,
|
||||
}: {
|
||||
text: string;
|
||||
length: number;
|
||||
offset: number;
|
||||
}) {
|
||||
const clip_uuid = uuid.v4();
|
||||
const newClip = {
|
||||
version: 1,
|
||||
type: ClipType.PROMPT_TRAVEL,
|
||||
group: ClipGroup.PROMPT_TRAVEL,
|
||||
offset,
|
||||
length: length,
|
||||
name: text,
|
||||
clip_uuid,
|
||||
} as Clip;
|
||||
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.ADD_CLIP,
|
||||
data: newClip,
|
||||
});
|
||||
|
||||
const oldPromptTravelGroup = promptTravelGroup.value;
|
||||
promptTravelGroup.value = {
|
||||
...oldPromptTravelGroup,
|
||||
clips: [...oldPromptTravelGroup.clips, newClip],
|
||||
};
|
||||
}
|
||||
|
||||
export function deletePromptTravelClip(clip: Clip) {
|
||||
const oldPromptTravelGroup = promptTravelGroup.value;
|
||||
promptTravelGroup.value = {
|
||||
...oldPromptTravelGroup,
|
||||
clips: [
|
||||
...oldPromptTravelGroup.clips.filter((row) => {
|
||||
if (row.clip_uuid === clip.clip_uuid) {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.DELETE_CLIP,
|
||||
data: row,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function loadPromptTravelData(item: ClipUI) {
|
||||
const existingPromptTravelId = promptTravelGroup.value;
|
||||
const newItem = {
|
||||
version: item.version,
|
||||
clip_uuid: item.clip_uuid,
|
||||
type: item.type,
|
||||
group: item.group,
|
||||
object_uuid: item.object_uuid,
|
||||
media_id: item.media_id,
|
||||
name: item.name,
|
||||
offset: item.offset,
|
||||
length: item.length,
|
||||
} as Clip;
|
||||
existingPromptTravelId.clips.push(newItem);
|
||||
existingPromptTravelId.clips.sort(
|
||||
(clipA, clipB) => clipA.offset - clipB.offset,
|
||||
);
|
||||
}
|
||||
@@ -1,65 +1,3 @@
|
||||
import { signal, computed } from "@preact/signals-core";
|
||||
import { characterGroup } from "~/pages/PageEnigma/signals/characterGroups";
|
||||
import { objectGroup } from "~/pages/PageEnigma/signals/objectGroup";
|
||||
import { pageWidth } from "~/signals";
|
||||
import { timelineScrollX } from "~/pages/PageEnigma/signals/timing";
|
||||
import { signal } from "@preact/signals-core";
|
||||
|
||||
// timeline
|
||||
export const scale = signal(0.05);
|
||||
export const filmLength = signal(7); // seconds
|
||||
export const timelineHeight = signal(0);
|
||||
|
||||
export const cameraMinimized = signal(false);
|
||||
export const audioMinimized = signal(false);
|
||||
export const objectsMinimized = signal(false);
|
||||
|
||||
export const characterHeight = computed(() => {
|
||||
if (!characterGroup.value.characters.length) {
|
||||
return 0;
|
||||
}
|
||||
return (
|
||||
characterGroup.value.characters.reduce((totalHeight, character) => {
|
||||
return totalHeight + (character.minimized ? 47 : 211);
|
||||
}, 0) + 8
|
||||
);
|
||||
});
|
||||
|
||||
export const objectHeight = computed(() => {
|
||||
if (!objectGroup.value.objects.length) {
|
||||
return 0;
|
||||
}
|
||||
if (objectsMinimized.value) {
|
||||
return 55;
|
||||
}
|
||||
return 47 + objectGroup.value.objects.length * 47 + 8;
|
||||
});
|
||||
|
||||
export const fullHeight = computed(() => {
|
||||
return (
|
||||
characterHeight.value +
|
||||
objectHeight.value +
|
||||
(cameraMinimized.value ? 47 : 103) +
|
||||
(audioMinimized.value ? 47 : 103) +
|
||||
24
|
||||
);
|
||||
});
|
||||
|
||||
export const minimizeIconPosition = signal(20);
|
||||
|
||||
export const timelinePremiumLockPosition = computed(() => {
|
||||
return Math.min(
|
||||
pageWidth.value - 2000 + timelineScrollX.value,
|
||||
fullWidth.value - 24,
|
||||
);
|
||||
});
|
||||
|
||||
export const fullWidth = computed(() => {
|
||||
return filmLength.value * 1000 * 4 * scale.value;
|
||||
});
|
||||
|
||||
// side panel
|
||||
export const sidePanelWidth = signal(0);
|
||||
|
||||
export const frameTrackButtonWidthPx = 80;
|
||||
export const dragHandleWidth = 10;
|
||||
export const unitPx = 4; // Used for front-end dimension calculation between time and px
|
||||
@@ -1,68 +0,0 @@
|
||||
import { QueueKeyframe, Keyframe, MediaItem } from "~/pages/PageEnigma/models";
|
||||
import {
|
||||
addCameraKeyframe,
|
||||
addCharacterKeyframe,
|
||||
addObjectKeyframe,
|
||||
audioGroup,
|
||||
cameraGroup,
|
||||
characterGroup,
|
||||
deleteCameraKeyframe,
|
||||
deleteCharacter,
|
||||
deleteCharacterKeyframe,
|
||||
deleteObjectKeyframe,
|
||||
objectGroup,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
import { ClipGroup } from "~/enums";
|
||||
import { deleteObject } from "~/pages/PageEnigma/signals/objectGroup/deleteObject";
|
||||
|
||||
const ADD_KEYFRAME: Record<
|
||||
ClipGroup,
|
||||
(keyframe: QueueKeyframe, offset: number) => void
|
||||
> = {
|
||||
[ClipGroup.CAMERA]: addCameraKeyframe,
|
||||
[ClipGroup.CHARACTER]: addCharacterKeyframe,
|
||||
[ClipGroup.OBJECT]: addObjectKeyframe,
|
||||
[ClipGroup.GLOBAL_AUDIO]: () => {},
|
||||
[ClipGroup.PROMPT_TRAVEL]: () => {},
|
||||
};
|
||||
|
||||
const DELETE_KEYFRAME: Record<ClipGroup, (keyframe: Keyframe) => void> = {
|
||||
[ClipGroup.CAMERA]: deleteCameraKeyframe,
|
||||
[ClipGroup.CHARACTER]: deleteCharacterKeyframe,
|
||||
[ClipGroup.OBJECT]: deleteObjectKeyframe,
|
||||
[ClipGroup.GLOBAL_AUDIO]: () => {},
|
||||
[ClipGroup.PROMPT_TRAVEL]: () => {},
|
||||
};
|
||||
|
||||
export function addKeyframe(keyframe: QueueKeyframe, offset: number) {
|
||||
ADD_KEYFRAME[keyframe.group](keyframe, offset);
|
||||
}
|
||||
|
||||
export function deleteKeyframe(keyframe: Keyframe) {
|
||||
DELETE_KEYFRAME[keyframe.group](keyframe);
|
||||
}
|
||||
|
||||
export function clearExistingData() {
|
||||
characterGroup.value = {
|
||||
id: "ChG1",
|
||||
characters: [],
|
||||
};
|
||||
cameraGroup.value = {
|
||||
id: "CG1",
|
||||
keyframes: [],
|
||||
};
|
||||
audioGroup.value = {
|
||||
id: "AG-1",
|
||||
clips: [],
|
||||
muted: false,
|
||||
};
|
||||
objectGroup.value = {
|
||||
id: "OG1",
|
||||
objects: [],
|
||||
};
|
||||
}
|
||||
|
||||
export function deleteObjectOrCharacter(item: MediaItem) {
|
||||
deleteCharacter(item);
|
||||
deleteObject(item);
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import { signal } from "@preact/signals-core";
|
||||
|
||||
export const currentTime = signal(0);
|
||||
// TODO: scrubber time exists to sync the time the scrubber points
|
||||
// to with the time display on video controls, we could eliminate this
|
||||
// and calculate everything properly with currentTime, but that would
|
||||
// be a developer brain struggle for another day.
|
||||
export const pointerScrubber = signal(0);
|
||||
export const secondaryScrubber = signal(0);
|
||||
export const timelineScrollX = signal(0);
|
||||
export const timelineScrollY = signal(0);
|
||||
export const stylizeScrollX = signal(0);
|
||||
@@ -1,25 +0,0 @@
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
import { filmLength } from "~/pages/PageEnigma/signals";
|
||||
import { MediaItem, Clip } from "~/pages/PageEnigma/models";
|
||||
|
||||
export function publishClip(
|
||||
newClip: Clip,
|
||||
dragItem: MediaItem,
|
||||
offset: number,
|
||||
) {
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.ADD_CLIP,
|
||||
data: newClip,
|
||||
});
|
||||
if (offset + (dragItem.length ?? 0) > filmLength.value * 1000) {
|
||||
newClip.length = filmLength.value * 1000 - offset;
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.UPDATE_CLIP,
|
||||
data: newClip,
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user