From 4898233e52ea554339cdfaeb7c0277aa45584488 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Thu, 4 Apr 2024 08:27:08 +0200 Subject: [PATCH 1/5] More refactor/cleanup --- .../comps/SidePanelTabs/AnimationElement.tsx | 4 +- .../comps/SidePanelTabs/AnimationElements.tsx | 7 ++- .../comps/SidePanelTabs/AudioElement.tsx | 4 +- .../PageEnigma/comps/Timeline/Character.tsx | 19 +++--- .../PageEnigma/comps/Timeline/ObjectTrack.tsx | 3 +- .../comps/{Scrubber => Timeline}/Scrubber.tsx | 0 .../PageEnigma/comps/Timeline/Timeline.tsx | 2 +- .../PageEnigma/comps/Timeline/TrackClip.tsx | 10 +-- .../PageEnigma/comps/Timeline/TrackClips.tsx | 63 +++++++++++-------- .../comps/Timeline/TrackKeyFrame.tsx | 19 ++---- .../comps/Timeline/TrackKeyFrames.tsx | 25 ++++---- .../Timeline/utils/useMouseEventsClip.ts | 10 ++- .../Timeline/utils/useMouseEventsKeyframe.ts | 16 ++--- .../contexts/TrackContext/TrackContext.tsx | 3 +- .../datastructures/clips/clip_ui.ts | 44 ++++++------- app/pages/PageEnigma/js/api_manager.ts | 60 +++++++++--------- app/pages/PageEnigma/js/timeline.ts | 5 +- app/pages/PageEnigma/store/cameraGroup.ts | 11 +++- app/pages/PageEnigma/store/characterGroups.ts | 29 ++++++--- app/pages/PageEnigma/store/dragAndDrop.ts | 3 +- app/pages/PageEnigma/store/objectGroup.ts | 39 +++++++----- 21 files changed, 209 insertions(+), 167 deletions(-) rename app/pages/PageEnigma/comps/{Scrubber => Timeline}/Scrubber.tsx (100%) diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElement.tsx index 1520f19497..284c1245da 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElement.tsx @@ -1,5 +1,5 @@ import { useCallback, useContext, useEffect } from "react"; -import { MediaClip } from "~/pages/PageEnigma/models/track"; +import { ClipType, MediaClip } from "~/pages/PageEnigma/models/track"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import { canDrop, @@ -11,7 +11,7 @@ import { useSignals } from "@preact/signals-react/runtime"; interface Props { clip: MediaClip; - type: "animations" | "audio"; + type: ClipType; } export const AnimationElement = ({ clip, type }: Props) => { diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElements.tsx index c9cbceaf51..9ef9b7eb71 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElements.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AnimationElements.tsx @@ -3,6 +3,7 @@ import { AnimationElement } from "./AnimationElement"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import { useSignals } from "@preact/signals-react/runtime"; import { timelineHeight } from "~/pages/PageEnigma/store"; +import { ClipType } from "~/pages/PageEnigma/models/track"; export const AnimationElements = () => { const { animationClips } = useContext(TrackContext); @@ -17,7 +18,11 @@ export const AnimationElements = () => {
{animationClips.map((clip) => { return ( - + ); })}
diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx index 9da211d7ee..ca444a10e3 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx @@ -1,5 +1,5 @@ import { useCallback, useContext, useEffect } from "react"; -import { MediaClip } from "~/pages/PageEnigma/models/track"; +import { ClipType, MediaClip } from "~/pages/PageEnigma/models/track"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import { canDrop, @@ -11,7 +11,7 @@ import { useSignals } from "@preact/signals-react/runtime"; interface Props { clip: MediaClip; - type: "animations" | "audio"; + type: ClipType; } export const AudioElement = ({ clip, type }: Props) => { diff --git a/app/pages/PageEnigma/comps/Timeline/Character.tsx b/app/pages/PageEnigma/comps/Timeline/Character.tsx index cf10ea6d9c..ffe1d229e0 100644 --- a/app/pages/PageEnigma/comps/Timeline/Character.tsx +++ b/app/pages/PageEnigma/comps/Timeline/Character.tsx @@ -7,10 +7,11 @@ import { updateCharacters, } from "~/pages/PageEnigma/store"; import { TrackKeyFrames } from "~/pages/PageEnigma/comps/Timeline/TrackKeyFrames"; +import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; function buildUpdaters( updateCharacters: (options: { - type: "animations" | "positions" | "lipSync"; + type: ClipType; id: string; length: number; offset: number; @@ -21,17 +22,17 @@ function buildUpdaters( length: number; offset: number; }) { - updateCharacters({ ...options, type: "animations" }); + updateCharacters({ ...options, type: ClipType.ANIMATION }); } function updateClipPosition(options: { id: string; offset: number }) { - updateCharacters({ ...options, length: 0, type: "positions" }); + updateCharacters({ ...options, length: 0, type: ClipType.TRANSFORM }); } function updateClipLipSync(options: { id: string; length: number; offset: number; }) { - updateCharacters({ ...options, type: "lipSync" }); + updateCharacters({ ...options, type: ClipType.AUDIO }); } return { updateClipLipSync, updateClipPosition, updateClipAnimations }; } @@ -68,15 +69,15 @@ export const Character = ({ characterId }: Props) => { clips={animationClips} title="Animation" updateClip={updateClipAnimations} - style="character" - type="animations" + group={ClipGroup.CHARACTER} + type={ClipType.ANIMATION} /> { updateClip={updateClipLipSync} muted={character.muted} toggleMute={toggleCharacterLipSyncMute} - style="character" - type="lipSync" + group={ClipGroup.CHARACTER} + type={ClipType.AUDIO} /> diff --git a/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx b/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx index a63c6f4e2d..32e90f9959 100644 --- a/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx +++ b/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx @@ -1,5 +1,6 @@ import { fullWidth, objectGroup, updateObject } from "~/pages/PageEnigma/store"; import { TrackKeyFrames } from "~/pages/PageEnigma/comps/Timeline/TrackKeyFrames"; +import { ClipGroup } from "~/pages/PageEnigma/models/track"; export const ObjectTrack = () => { return ( @@ -17,7 +18,7 @@ export const ObjectTrack = () => { keyframes={object.keyframes} title={`${object.name} Position/Rotation`} updateKeyframe={updateObject} - style="objects" + group={ClipGroup.OBJECT} /> ))} diff --git a/app/pages/PageEnigma/comps/Scrubber/Scrubber.tsx b/app/pages/PageEnigma/comps/Timeline/Scrubber.tsx similarity index 100% rename from app/pages/PageEnigma/comps/Scrubber/Scrubber.tsx rename to app/pages/PageEnigma/comps/Timeline/Scrubber.tsx diff --git a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx index 3de3cb7155..5ceb0ae6db 100644 --- a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx +++ b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx @@ -13,7 +13,7 @@ import { import { useQueueHandler } from "~/pages/PageEnigma/comps/Timeline/utils/useQueueHandler"; import { useSignals } from "@preact/signals-react/runtime"; import { TimerGrid } from "~/pages/PageEnigma/comps/TimerGrid/TimerGrid"; -import { Scrubber } from "~/pages/PageEnigma/comps/Scrubber/Scrubber"; +import { Scrubber } from "~/pages/PageEnigma/comps/Timeline/Scrubber"; import { Characters } from "~/pages/PageEnigma/comps/Timeline/Characters"; import { ObjectGroups } from "~/pages/PageEnigma/comps/Timeline/ObjectGroups"; import useUpdateKeyframe from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateKeyframe"; diff --git a/app/pages/PageEnigma/comps/Timeline/TrackClip.tsx b/app/pages/PageEnigma/comps/Timeline/TrackClip.tsx index d12685ac28..64532b0e2e 100644 --- a/app/pages/PageEnigma/comps/Timeline/TrackClip.tsx +++ b/app/pages/PageEnigma/comps/Timeline/TrackClip.tsx @@ -1,17 +1,17 @@ import { useMouseEventsClip } from "./utils/useMouseEventsClip"; import { useState } from "react"; import { scale, selectedItem } from "~/pages/PageEnigma/store"; -import { Clip } from "~/pages/PageEnigma/models/track"; +import { Clip, ClipGroup } from "~/pages/PageEnigma/models/track"; interface Props { min: number; max: number; - style: "character" | "camera" | "audio" | "objects"; + group: ClipGroup; clip: Clip; updateClip: (options: { id: string; offset: number; length: number }) => void; } -export const TrackClip = ({ clip, min, max, style, updateClip }: Props) => { +export const TrackClip = ({ clip, min, max, group, updateClip }: Props) => { const [state, setState] = useState({ length: clip.length, offset: clip.offset, @@ -31,8 +31,8 @@ export const TrackClip = ({ clip, min, max, style, updateClip }: Props) => { const classes = [ "absolute", clip.clip_uuid === selectedClipId - ? `bg-${style}-selected` - : `bg-${style}-clip`, + ? `bg-${group}-selected` + : `bg-${group}-clip`, ]; return ( diff --git a/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx b/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx index 565944df23..6d45a2f9c6 100644 --- a/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx +++ b/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx @@ -1,7 +1,7 @@ import { TrackClip } from "~/pages/PageEnigma/comps/Timeline/TrackClip"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faVolume, faVolumeSlash } from "@fortawesome/pro-solid-svg-icons"; -import { Clip } from "~/pages/PageEnigma/models/track"; +import { Clip, ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; import { PointerEvent } from "react"; import { canDrop, @@ -17,13 +17,38 @@ interface Props { id: string; clips: Clip[]; title: string; - style: "character" | "audio" | "camera" | "objects"; - type?: "animations" | "positions" | "lipSync"; + group: ClipGroup; + type?: ClipType; toggleMute?: () => void; muted?: boolean; updateClip: (options: { id: string; length: number; offset: number }) => void; } +function getCanBuild({ + dragType, + type, + group, +}: { + dragType: ClipType | null; + type?: ClipType; + group: ClipGroup; +}) { + if (dragType === ClipType.ANIMATION) { + if (dragType === type) { + return true; + } + } + if (dragType === ClipType.AUDIO) { + if (group === ClipGroup.CHARACTER && type === ClipType.AUDIO) { + return true; + } + if (group === ClipGroup.GLOBAL_AUDIO) { + return true; + } + } + return false; +} + export const TrackClips = ({ id, clips, @@ -31,36 +56,22 @@ export const TrackClips = ({ updateClip, muted, title, - style, + group, type, }: Props) => { - const trackType = type ?? style; + const trackType = (type ?? group) as ClipType; function onPointerOver() { - if (dragType.value === "animations") { - if (dragType.value !== trackType) { - return; - } + if (getCanBuild({ dragType: dragType.value, type, group })) { + dropId.value = id; } - if (dragType.value === "audio") { - if (["lipSync", "audio"].indexOf(trackType) === -1) { - return; - } - } - dropId.value = id; } function onPointerMove(event: PointerEvent) { - if (dragType.value === "animations") { - if (dragType.value !== trackType) { - return; - } - } - if (dragType.value === "audio") { - if (["lipSync", "audio"].indexOf(trackType) === -1) { - return; - } + if (!getCanBuild({ dragType: dragType.value, type, group })) { + return; } + const track = document.getElementById(`track-${trackType}-${id}`); if (!track) { return; @@ -101,7 +112,7 @@ export const TrackClips = ({
diff --git a/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx b/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx index c828baebc0..2dbf099e8e 100644 --- a/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx +++ b/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx @@ -1,28 +1,21 @@ import { Keyframe } from "~/pages/PageEnigma/models/track"; import { useState } from "react"; -import { scale, selectedItem } from "~/pages/PageEnigma/store"; +import { filmLength, scale, selectedItem } from "~/pages/PageEnigma/store"; import { useMouseEventsKeyframe } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faSortUpDown } from "@fortawesome/pro-solid-svg-icons"; +import { useSignals } from "@preact/signals-react/runtime"; interface Props { - min: number; - max: number; keyframe: Keyframe; updateKeyframe: (options: { id: string; offset: number }) => void; } -export const TrackKeyFrame = ({ - keyframe, - min, - max, - updateKeyframe, -}: Props) => { +export const TrackKeyFrame = ({ keyframe, updateKeyframe }: Props) => { + useSignals(); const [offset, setOffset] = useState(keyframe.offset); const { onPointerDown } = useMouseEventsKeyframe({ keyframe, - max, - min, + max: filmLength.value * 60, + min: 0, updateKeyframe, setOffset, }); diff --git a/app/pages/PageEnigma/comps/Timeline/TrackKeyFrames.tsx b/app/pages/PageEnigma/comps/Timeline/TrackKeyFrames.tsx index aea6f7334c..ca1316a033 100644 --- a/app/pages/PageEnigma/comps/Timeline/TrackKeyFrames.tsx +++ b/app/pages/PageEnigma/comps/Timeline/TrackKeyFrames.tsx @@ -1,15 +1,18 @@ -import { Keyframe } from "~/pages/PageEnigma/models/track"; -import { filmLength } from "~/pages/PageEnigma/store"; +import { ClipGroup, Keyframe } from "~/pages/PageEnigma/models/track"; import { TrackKeyFrame } from "~/pages/PageEnigma/comps/Timeline/TrackKeyFrame"; interface Props { id: string; keyframes: Keyframe[]; title: string; - style: "character" | "audio" | "camera" | "objects"; + group: ClipGroup; toggleMute?: () => void; muted?: boolean; - updateKeyframe: (options: { id: string; offset: number }) => void; + updateKeyframe: (options: { + id: string; + offset: number; + force?: boolean; + }) => void; } export const TrackKeyFrames = ({ @@ -17,23 +20,17 @@ export const TrackKeyFrames = ({ keyframes, updateKeyframe, title, - style, + group, }: Props) => { return (
- {keyframes.map((keyframe, index) => ( + {keyframes.map((keyframe) => ( 0 ? keyframes[index - 1].offset + 1 : 0} - max={ - index < keyframes.length - 1 - ? keyframes[index + 1].offset - : filmLength.value * 60 - } updateKeyframe={updateKeyframe} keyframe={keyframe} /> diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts index 1dbe7ed124..4746501ee8 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts @@ -37,13 +37,11 @@ export const useMouseEventsClip = ( if (isActive.current === "drag") { if (deltaOffset < min) { currOffset.current = min; - return; + } else if (deltaOffset + currLength.current > max) { + currOffset.current = max; + } else { + currOffset.current = deltaOffset; } - if (deltaOffset + currLength.current > max) { - currOffset.current = max - currLength.current; - return; - } - currOffset.current = deltaOffset; } if (isActive.current === "left") { if (initLength.current - delta < 30 || deltaOffset < min) { diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe.ts index 20f46e2e5f..8fde4ec267 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe.ts @@ -1,6 +1,10 @@ import React, { Dispatch, useCallback, useEffect, useRef } from "react"; import { Keyframe } from "~/pages/PageEnigma/models/track"; -import { canDrop, scale } from "~/pages/PageEnigma/store"; +import { + canDrop, + scale, + showDuplicateOffsetWarning, +} from "~/pages/PageEnigma/store"; import { useSignals } from "@preact/signals-react/runtime"; export const useMouseEventsKeyframe = ({ @@ -40,15 +44,13 @@ export const useMouseEventsKeyframe = ({ if (isActive.current === "drag") { if (deltaOffset < min) { currOffset.current = min; - return; - } - if (deltaOffset > max) { + } else if (deltaOffset > max) { currOffset.current = max; - return; + } else { + currOffset.current = deltaOffset; } - currOffset.current = deltaOffset; + setOffset(currOffset.current); } - setOffset(currOffset.current); }, [max, min, setOffset], ); diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx index adc2738f8e..b3038d6be3 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx @@ -3,6 +3,7 @@ import { MediaClip, Keyframe, QueueKeyframe, + ClipType, } from "~/pages/PageEnigma/models/track"; export const TrackContext = createContext<{ @@ -15,7 +16,7 @@ export const TrackContext = createContext<{ audioClips: MediaClip[]; // drag and drop - startDrag: (type: "animations" | "audio", id: string, length: number) => void; + startDrag: (type: ClipType, id: string, length: number) => void; endDrag: () => void; }>({ addKeyframe: () => {}, diff --git a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts index eb06cc2eba..04dce4428c 100644 --- a/app/pages/PageEnigma/datastructures/clips/clip_ui.ts +++ b/app/pages/PageEnigma/datastructures/clips/clip_ui.ts @@ -1,47 +1,49 @@ // Clip offsets represent the state of the clip on the timeline as well as what type of clip it is. // it is created from a media id. +import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; + export class ClipUI { - version: number - type: "transform" | "audio" | "animation" | "lipsync" - group: "character" | "object" | "camera" | "global_audio" - name: string - media_id: string - object_uuid: string - offset: number // in frames - length: number // in frames + version: number; + type: ClipType; + group: ClipGroup; + name: string; + media_id: string; + object_uuid: string; + offset: number; // in frames + length: number; // in frames constructor( version: number, - type: "transform" | "audio" | "animation" | "lipsync", - group: "character" | "object" | "camera" | "global_audio", + type: ClipType, + group: ClipGroup, name: string, media_id: string, object_uuid: string, offset: number, length: number, ) { - this.version = version - this.group = group // Only needed for UI + this.version = version; + this.group = group; // Only needed for UI - this.name = name // UI - this.type = type // UI and Animation / Audio / Lipsync / : Engine + this.name = name; // UI + this.type = type; // UI and Animation / Audio / Lipsync / : Engine - this.object_uuid = object_uuid // Animation / Audio / Lipsync / : Engine - this.media_id = media_id // Animation / Audio / Lipsync / : Engine - this.offset = offset - this.length = length + this.object_uuid = object_uuid; // Animation / Audio / Lipsync / : Engine + this.media_id = media_id; // Animation / Audio / Lipsync / : Engine + this.offset = offset; + this.length = length; } toJSON(): string { return JSON.stringify({ version: this.version, - group:this.group, + group: this.group, name: this.name, type: this.type, object_uuid: this.object_uuid, media_id: this.media_id, start_offset: this.offset, - ending_offset: this.length - }) + ending_offset: this.length, + }); } } diff --git a/app/pages/PageEnigma/js/api_manager.ts b/app/pages/PageEnigma/js/api_manager.ts index f9e61c42f0..fde45ddd37 100644 --- a/app/pages/PageEnigma/js/api_manager.ts +++ b/app/pages/PageEnigma/js/api_manager.ts @@ -54,9 +54,9 @@ export enum Visibility { */ type Data = { [key: string]: any }; class APIManagerResponseSuccess { - public user_message: String; + public user_message: string; public data: Data | null; - constructor(user_message: String = "", data: Data | null = null) { + constructor(user_message: string = "", data: Data | null = null) { this.data = data; this.user_message = user_message; } @@ -74,7 +74,7 @@ class APIManagerResponseError extends Error { } export class APIManager { - baseUrl: String; + baseUrl: string; constructor() { this.baseUrl = "https://api.fakeyou.com"; @@ -139,9 +139,9 @@ export class APIManager { throw new APIManagerResponseError("Failed to download file"); } // Convert the response from a blob to json text - let blob = await file_response.blob(); + const blob = await file_response.blob(); const json_result: string = await new Promise((resolve, reject) => { - let reader = new FileReader(); + const reader = new FileReader(); reader.onloadend = () => resolve(JSON.parse(reader.result as string)); reader.onerror = reject; reader.readAsText(blob); @@ -156,7 +156,7 @@ export class APIManager { scene_glb_media_file_token, ); console.log(`GLB ${media_bucket_path}`); - let glbLoader = new GLTFLoader(); + const glbLoader = new GLTFLoader(); // promisify this const loadGlb = ( bucket_path: string, @@ -188,13 +188,13 @@ export class APIManager { * @returns */ private async getMediaFile(media_file_token: string): Promise { - let api_base_url = "https://api.fakeyou.com"; - let url = `${api_base_url}/v1/media_files/file/${media_file_token}`; - let response = await fetch(url); - let json = await JSON.parse(await response.text()); - let bucketPath = json["media_file"]["public_bucket_path"]; - let media_base_url = "https://storage.googleapis.com/vocodes-public"; - let media_url = `${media_base_url}${bucketPath}`; // gets you a bucket path + const api_base_url = "https://api.fakeyou.com"; + const url = `${api_base_url}/v1/media_files/file/${media_file_token}`; + const response = await fetch(url); + const json = await JSON.parse(await response.text()); + const bucketPath = json["media_file"]["public_bucket_path"]; + const media_base_url = "https://storage.googleapis.com/vocodes-public"; + const media_url = `${media_base_url}${bucketPath}`; // gets you a bucket path return media_url; } @@ -218,7 +218,7 @@ export class APIManager { scene_glb_media_file_token: string | null, ): Promise { const url = `${this.baseUrl}/v1/media_files/write/engine_asset`; - let uuid = uuidv4(); + const uuid = uuidv4(); const form_data = new FormData(); form_data.append("uuid_idempotency_token", uuid); @@ -257,7 +257,7 @@ export class APIManager { timeline_state: TimelineDataState | null, // only for now. ): Promise { const url = `${this.baseUrl}/v1/media_files/write/scene_file`; - let uuid = uuidv4(); + const uuid = uuidv4(); console.log( `Saving Scene scene_media_file_token:${scene_media_file_token} | scene_glb_media_file_token:${scene_glb_media_file_token}`, @@ -311,7 +311,7 @@ export class APIManager { fileName: string, ): Promise { const url = `${this.baseUrl}/v1/media_uploads/upload`; - let uuid = uuidv4(); + const uuid = uuidv4(); const formData = new FormData(); formData.append("uuid_idempotency_token", uuid); @@ -346,11 +346,11 @@ export class APIManager { negative_prompt: string, ): Promise { const url = `https://funnel.tailce84f.ts.net/preview/`; - + const payload = { style: style, positive_prompt: positive_prompt, - negative_prompt: negative_prompt + negative_prompt: negative_prompt, }; const formData = new FormData(); @@ -393,14 +393,15 @@ export class APIManager { return result; } - - public async stylizeVideo(media_token: string, - style: ArtStyle, - positive_prompt:string, - negative_prompt:string, - visibility:Visibility) { - let uuid = uuidv4(); - + public async stylizeVideo( + media_token: string, + style: ArtStyle, + positive_prompt: string, + negative_prompt: string, + visibility: Visibility, + ) { + const uuid = uuidv4(); + const data = { uuid_idempotency_token: uuid, style: style, @@ -410,10 +411,10 @@ export class APIManager { trim_start_millis: 0, trim_end_millis: 3000, enable_lipsync: true, - creator_set_visibility: visibility - } + creator_set_visibility: visibility, + }; - const json_data = JSON.stringify(data) + const json_data = JSON.stringify(data); const response = await fetch(`${this.baseUrl}/v1/video/enqueue_vst`, { method: "POST", @@ -435,4 +436,3 @@ export class APIManager { return await response.json(); } } - diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 31c73a78b5..3bf8545c4b 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -11,6 +11,7 @@ import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "../Queue/QueueNames"; import { toEngineActions } from "../Queue/toEngineActions"; import { fromEngineActions } from "../Queue/fromEngineActions"; +import { ClipType } from "~/pages/PageEnigma/models/track"; // Every object uuid / entity has a track. export class TimelineCurrentState { @@ -169,10 +170,10 @@ export class TimeLine { ); switch (type) { - case "animation": + case ClipType.ANIMATION: this.animation_engine.load_object(object_uuid, media_id, name); break; - case "transform": + case ClipType.TRANSFORM: this.transform_engine.loadObject(object_uuid, data["data"]["length"]); break; } diff --git a/app/pages/PageEnigma/store/cameraGroup.ts b/app/pages/PageEnigma/store/cameraGroup.ts index 1330f9b872..093e751eb2 100644 --- a/app/pages/PageEnigma/store/cameraGroup.ts +++ b/app/pages/PageEnigma/store/cameraGroup.ts @@ -13,6 +13,11 @@ export const cameraGroup = signal({ id: "CG1", keyframes: [] }); export function updateCamera({ id, offset }: { id: string; offset: number }) { const oldCameraGroup = cameraGroup.value; + + if (oldCameraGroup.keyframes.some((row) => row.offset === offset)) { + return; + } + const newKeyframes = [...oldCameraGroup.keyframes]; const keyframe = newKeyframes.find((row) => row.keyframe_uuid === id); if (!keyframe) { @@ -33,7 +38,10 @@ export function updateCamera({ id, offset }: { id: string; offset: number }) { } export function addCameraKeyframe(keyframe: QueueKeyframe, offset: number) { - console.log("cam", keyframe); + const oldCameraGroup = cameraGroup.value; + if (oldCameraGroup.keyframes.some((row) => row.offset === offset)) { + return; + } const newKeyframe = { version: keyframe.version, keyframe_uuid: uuid.v4(), @@ -46,7 +54,6 @@ export function addCameraKeyframe(keyframe: QueueKeyframe, offset: number) { selected: false, } as Keyframe; - const oldCameraGroup = cameraGroup.value; cameraGroup.value = { ...oldCameraGroup, keyframes: [...oldCameraGroup.keyframes, newKeyframe].sort( diff --git a/app/pages/PageEnigma/store/characterGroups.ts b/app/pages/PageEnigma/store/characterGroups.ts index 3d972fe95f..8bed1c6436 100644 --- a/app/pages/PageEnigma/store/characterGroups.ts +++ b/app/pages/PageEnigma/store/characterGroups.ts @@ -28,14 +28,16 @@ export function updateCharacters({ id, offset, length, + force, }: { - type: "animations" | "positions" | "lipSync"; + type: ClipType; id: string; - length: number; + length?: number; offset: number; + force?: boolean; }) { const oldCharacterGroups = characterGroups.value; - if (type === "animations") { + if (type === ClipType.ANIMATION) { characterGroups.value = oldCharacterGroups.map((character) => { const newAnimationClips = [...character.animationClips]; const clipIndex = newAnimationClips.findIndex( @@ -46,7 +48,7 @@ export function updateCharacters({ } const clip = newAnimationClips[clipIndex]; clip.offset = offset; - clip.length = length; + clip.length = length!; Queue.publish({ queueName: QueueNames.TO_ENGINE, @@ -61,7 +63,7 @@ export function updateCharacters({ }); } - if (type === "positions") { + if (type === ClipType.TRANSFORM) { characterGroups.value = oldCharacterGroups.map((character) => { const newPositionKeyframes = [...character.positionKeyframes]; const keyframeIndex = newPositionKeyframes.findIndex( @@ -85,7 +87,7 @@ export function updateCharacters({ }; }); } - if (type === "lipSync") { + if (type === ClipType.AUDIO) { characterGroups.value = oldCharacterGroups.map((character) => { const newLipSyncClips = [...character.lipSyncClips]; const clipIndex = newLipSyncClips.findIndex( @@ -96,7 +98,7 @@ export function updateCharacters({ } const clip = newLipSyncClips[clipIndex]; clip.offset = offset; - clip.length = length; + clip.length = length!; Queue.publish({ queueName: QueueNames.TO_ENGINE, @@ -204,6 +206,18 @@ export function addCharacterAudio({ } export function addCharacterKeyframe(keyframe: QueueKeyframe, offset: number) { + const oldCharacterGroups = characterGroups.value; + + if ( + oldCharacterGroups.some((characterGroup) => { + return characterGroup.positionKeyframes.some( + (row) => row.offset === offset, + ); + }) + ) { + return; + } + const newKeyframe = { version: keyframe.version, keyframe_uuid: uuid.v4(), @@ -216,7 +230,6 @@ export function addCharacterKeyframe(keyframe: QueueKeyframe, offset: number) { selected: false, } as Keyframe; - const oldCharacterGroups = characterGroups.value; characterGroups.value = oldCharacterGroups.map((character) => { if (character.id !== keyframe.object_uuid) { return { ...character }; diff --git a/app/pages/PageEnigma/store/dragAndDrop.ts b/app/pages/PageEnigma/store/dragAndDrop.ts index 0f286f36d6..ba3d1bec9a 100644 --- a/app/pages/PageEnigma/store/dragAndDrop.ts +++ b/app/pages/PageEnigma/store/dragAndDrop.ts @@ -1,6 +1,7 @@ import { signal } from "@preact/signals-core"; +import { ClipType } from "~/pages/PageEnigma/models/track"; -export const dragType = signal<"animations" | "audio" | null>(null); +export const dragType = signal(null); export const dragId = signal(null); export const canDrop = signal(false); export const overTimeline = signal(false); diff --git a/app/pages/PageEnigma/store/objectGroup.ts b/app/pages/PageEnigma/store/objectGroup.ts index f2778dad68..72304b0a86 100644 --- a/app/pages/PageEnigma/store/objectGroup.ts +++ b/app/pages/PageEnigma/store/objectGroup.ts @@ -15,16 +15,24 @@ export const objectGroup = signal({ }); export function updateObject({ id, offset }: { id: string; offset: number }) { - const oldObject = objectGroup.value; + const oldObjectGroup = objectGroup.value; + const obj = oldObjectGroup.objects.find((objectTrack) => + objectTrack.keyframes.some((row) => row.keyframe_uuid === id), + ); + + if (obj && obj.keyframes.some((row) => row.offset === offset)) { + return; + } + objectGroup.value = { - id: oldObject.id, - objects: oldObject.objects.map((object) => ({ + id: oldObjectGroup.id, + objects: oldObjectGroup.objects.map((object) => ({ object_uuid: object.object_uuid, name: object.name, - keyframes: object.keyframes.map((keyFrame) => { - if (keyFrame.keyframe_uuid !== id) { + keyframes: object.keyframes.map((keyframe) => { + if (keyframe.keyframe_uuid !== id) { return { - ...keyFrame, + ...keyframe, }; } @@ -32,13 +40,13 @@ export function updateObject({ id, offset }: { id: string; offset: number }) { queueName: QueueNames.TO_ENGINE, action: toEngineActions.UPDATE_KEYFRAME, data: { - ...keyFrame, + ...keyframe, offset, }, }); return { - ...keyFrame, + ...keyframe, offset, }; }), @@ -52,6 +60,10 @@ export function addObjectKeyframe(keyframe: QueueKeyframe, offset: number) { (row) => row.object_uuid === keyframe.object_uuid, ); + if (obj && obj.keyframes.some((row) => row.offset === offset)) { + return; + } + const newObject = obj ?? { object_uuid: keyframe.object_uuid, name: keyframe.object_name ?? "unknown", @@ -69,19 +81,16 @@ export function addObjectKeyframe(keyframe: QueueKeyframe, offset: number) { selected: false, } as Keyframe; newObject.keyframes.push(newKeyframe); - console.log(newObject); + newObject.keyframes.sort( (keyframeA, keyframeB) => keyframeA.offset - keyframeB.offset, ); objectGroup.value = { ...oldObjectGroup, - objects: [ - ...oldObjectGroup.objects.filter( - (object) => object.object_uuid !== keyframe.object_uuid, - ), - newObject, - ].sort((objA, objB) => (objA.object_uuid < objB.object_uuid ? -1 : 1)), + objects: [...oldObjectGroup.objects, newObject].sort((objA, objB) => + objA.object_uuid < objB.object_uuid ? -1 : 1, + ), }; Queue.publish({ queueName: QueueNames.TO_ENGINE, From 988e5b23f52f8605f006ffdd2c7576575b90b498 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Thu, 4 Apr 2024 10:37:56 +0200 Subject: [PATCH 2/5] More cleanup, bug fixing --- app/pages/PageEnigma/PageEnigma.tsx | 46 ++++++++++++++++ .../comps/SidePanelTabs/AudioElements.tsx | 7 ++- app/pages/PageEnigma/comps/Timeline/Audio.tsx | 5 +- .../PageEnigma/comps/Timeline/Camera.tsx | 3 +- .../comps/Timeline/ObjectGroups.tsx | 1 + .../PageEnigma/comps/Timeline/ObjectTrack.tsx | 2 +- .../comps/Timeline/TrackKeyFrame.tsx | 19 ++++--- .../Timeline/utils/useMouseEventsKeyframe.ts | 16 +++--- .../contexts/TrackContext/TrackProvider.tsx | 6 +-- .../TrackContext/utils/useUpdateDragDrop.ts | 3 +- .../TrackContext/utils/useUpdateKeyframe.ts | 1 - app/pages/PageEnigma/store/objectGroup.ts | 38 ++++++++++++-- package-lock.json | 27 +++++++++- package.json | 1 + tailwind.config.ts | 52 ++++++++++--------- 15 files changed, 169 insertions(+), 58 deletions(-) diff --git a/app/pages/PageEnigma/PageEnigma.tsx b/app/pages/PageEnigma/PageEnigma.tsx index da3f729918..4481f413b3 100644 --- a/app/pages/PageEnigma/PageEnigma.tsx +++ b/app/pages/PageEnigma/PageEnigma.tsx @@ -3,12 +3,58 @@ import { TrackProvider } from "~/pages/PageEnigma/contexts/TrackContext/TrackPro import { AppUIProvider } from "~/pages/PageEnigma/contexts/AppUiContext"; import { EngineProvider } from "~/pages/PageEnigma/contexts/EngineProvider"; import { DragComponent } from "~/pages/PageEnigma/comps/DragComponent/DragComponent"; +import { toast, ToastBar, Toaster } from "react-hot-toast"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faTriangleExclamation } from "@fortawesome/pro-solid-svg-icons"; export const PageEnigma = () => { return ( + + {(thisToast) => ( + + {({ message }) => { + return ( +
+
+ {thisToast.type === "error" && ( +
+ +
+ Warning! +
+
+ )} +
+ {message} +
+
+ +
+ ); + }} + + )} + diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx index a35bd2d552..ee3eb38341 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx @@ -3,6 +3,7 @@ import { AnimationElement } from "./AnimationElement"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import { useSignals } from "@preact/signals-react/runtime"; import { timelineHeight } from "~/pages/PageEnigma/store"; +import { ClipType } from "~/pages/PageEnigma/models/track"; export const AudioElements = () => { const { audioClips } = useContext(TrackContext); @@ -17,7 +18,11 @@ export const AudioElements = () => {
{audioClips.map((clip) => { return ( - + ); })}
diff --git a/app/pages/PageEnigma/comps/Timeline/Audio.tsx b/app/pages/PageEnigma/comps/Timeline/Audio.tsx index 3ab96ef4a1..81be75ce41 100644 --- a/app/pages/PageEnigma/comps/Timeline/Audio.tsx +++ b/app/pages/PageEnigma/comps/Timeline/Audio.tsx @@ -5,13 +5,14 @@ import { toggleAudioMute, updateAudio, } from "~/pages/PageEnigma/store"; +import { ClipGroup } from "~/pages/PageEnigma/models/track"; export const Audio = () => { const { clips } = audioGroup.value; return (
@@ -23,7 +24,7 @@ export const Audio = () => { clips={clips} title="Global Audio TrackClips" updateClip={updateAudio} - style="audio" + group={ClipGroup.GLOBAL_AUDIO} muted={audioGroup.value.muted} toggleMute={toggleAudioMute} /> diff --git a/app/pages/PageEnigma/comps/Timeline/Camera.tsx b/app/pages/PageEnigma/comps/Timeline/Camera.tsx index 473d108b74..27e7b12ffb 100644 --- a/app/pages/PageEnigma/comps/Timeline/Camera.tsx +++ b/app/pages/PageEnigma/comps/Timeline/Camera.tsx @@ -1,6 +1,7 @@ import { cameraGroup, fullWidth, updateCamera } from "~/pages/PageEnigma/store"; import { TrackKeyFrames } from "~/pages/PageEnigma/comps/Timeline/TrackKeyFrames"; import { useSignals } from "@preact/signals-react/runtime"; +import { ClipGroup } from "~/pages/PageEnigma/models/track"; export const Camera = () => { useSignals(); @@ -17,7 +18,7 @@ export const Camera = () => { id={cameraGroup.value.id} keyframes={keyframes} title="Camera Position/Rotation" - style="camera" + group={ClipGroup.CAMERA} updateKeyframe={updateCamera} />
diff --git a/app/pages/PageEnigma/comps/Timeline/ObjectGroups.tsx b/app/pages/PageEnigma/comps/Timeline/ObjectGroups.tsx index c32387d043..3e9e634c5d 100644 --- a/app/pages/PageEnigma/comps/Timeline/ObjectGroups.tsx +++ b/app/pages/PageEnigma/comps/Timeline/ObjectGroups.tsx @@ -4,6 +4,7 @@ import { ObjectTrack } from "~/pages/PageEnigma/comps/Timeline/ObjectTrack"; export const ObjectGroups = () => { useSignals(); + return ( <> {objectGroup.value.objects.length > 0 && ( diff --git a/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx b/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx index 32e90f9959..11ca18c6c0 100644 --- a/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx +++ b/app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx @@ -5,7 +5,7 @@ import { ClipGroup } from "~/pages/PageEnigma/models/track"; export const ObjectTrack = () => { return (
diff --git a/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx b/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx index 2dbf099e8e..71c7045eb6 100644 --- a/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx +++ b/app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx @@ -1,5 +1,4 @@ import { Keyframe } from "~/pages/PageEnigma/models/track"; -import { useState } from "react"; import { filmLength, scale, selectedItem } from "~/pages/PageEnigma/store"; import { useMouseEventsKeyframe } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe"; import { useSignals } from "@preact/signals-react/runtime"; @@ -11,27 +10,31 @@ interface Props { export const TrackKeyFrame = ({ keyframe, updateKeyframe }: Props) => { useSignals(); - const [offset, setOffset] = useState(keyframe.offset); - const { onPointerDown } = useMouseEventsKeyframe({ + const { onPointerDown, offset } = useMouseEventsKeyframe({ keyframe, max: filmLength.value * 60, min: 0, updateKeyframe, - setOffset, }); + const displayOffset = offset > -1 ? offset : keyframe.offset; + const selectedKeyframeId = (selectedItem.value as Keyframe | null)?.keyframe_uuid ?? ""; return (
); diff --git a/app/pages/PageEnigma/store/cameraGroup.ts b/app/pages/PageEnigma/store/cameraGroup.ts index 093e751eb2..71ca0b9e2f 100644 --- a/app/pages/PageEnigma/store/cameraGroup.ts +++ b/app/pages/PageEnigma/store/cameraGroup.ts @@ -8,13 +8,19 @@ import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; import * as uuid from "uuid"; import { signal } from "@preact/signals-core"; +import { toast } from "react-hot-toast"; export const cameraGroup = signal({ id: "CG1", keyframes: [] }); export function updateCamera({ id, offset }: { id: string; offset: number }) { const oldCameraGroup = cameraGroup.value; - if (oldCameraGroup.keyframes.some((row) => row.offset === offset)) { + const existingKeyframe = oldCameraGroup.keyframes.some((row) => { + return row.offset === offset && row.keyframe_uuid !== id; + }); + + if (existingKeyframe) { + toast.error("There can only be one keyframe at this offset."); return; } @@ -40,6 +46,7 @@ export function updateCamera({ id, offset }: { id: string; offset: number }) { export function addCameraKeyframe(keyframe: QueueKeyframe, offset: number) { const oldCameraGroup = cameraGroup.value; if (oldCameraGroup.keyframes.some((row) => row.offset === offset)) { + toast.error("There can only be one keyframe at this offset."); return; } const newKeyframe = { diff --git a/app/pages/PageEnigma/store/objectGroup.ts b/app/pages/PageEnigma/store/objectGroup.ts index f4fbe66274..c1361fe499 100644 --- a/app/pages/PageEnigma/store/objectGroup.ts +++ b/app/pages/PageEnigma/store/objectGroup.ts @@ -32,19 +32,11 @@ export function updateObject({ } const existingKeyframe = obj.keyframes.some((row) => { - console.log( - "upd", - row.keyframe_uuid, - row.offset, - offset, - id, - row.offset === offset && row.keyframe_uuid !== id, - ); return row.offset === offset && row.keyframe_uuid !== id; }); if (existingKeyframe) { - toast("There can only be one keyframe at an offset."); + toast.error("There can only be one keyframe at this offset."); return; } From 90097245112713f0936ef87e66915659a04f36ef Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:44:35 -0400 Subject: [PATCH 4/5] Restoring the old timeline code --- app/pages/PageEnigma/js/timeline.ts | 803 +++++++++++++-------------- app/pages/PageEnigma/models/track.ts | 1 + 2 files changed, 392 insertions(+), 412 deletions(-) diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 3c5c83a3e9..ee3cb50e14 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -12,432 +12,411 @@ import { QueueNames } from "../Queue/QueueNames"; import { toEngineActions } from "../Queue/toEngineActions"; import { fromEngineActions } from "../Queue/fromEngineActions"; import { ClipGroup, ClipType } from "~/pages/PageEnigma/models/track"; - // Every object uuid / entity has a track. export class TimelineCurrentState { - is_editable: boolean; - selected_object_ID: number; - constructor() { - this.is_editable = true; // can add clips to it - this.selected_object_ID = 0; - } + is_editable: boolean; + selected_object_ID: number; + constructor() { + this.is_editable = true; // can add clips to it + this.selected_object_ID = 0; + } } export class TimelineDataState { - timeline_items: ClipUI[]; - scrubber_frame_position: number; - constructor( - timeline_items: ClipUI[] = [], - scrubber_frame_position: number = 0, - ) { - this.timeline_items = timeline_items; - this.scrubber_frame_position = scrubber_frame_position; - } + timeline_items: ClipUI[]; + scrubber_frame_position: number; + constructor( + timeline_items: ClipUI[] = [], + scrubber_frame_position: number = 0, + ) { + this.timeline_items = timeline_items; + this.scrubber_frame_position = scrubber_frame_position; + } } export class TimeLine { - timeline_items: ClipUI[]; + timeline_items: ClipUI[]; - timeline_limit: number; - absolute_end: number; - scrubber_frame_position: number; - is_playing: boolean; + timeline_limit: number; + absolute_end: number; + scrubber_frame_position: number; + is_playing: boolean; - // plays audio - audio_engine: AudioEngine; - // key framing - transform_engine: TransformEngine; - // animation engine - animation_engine: AnimationEngine; - // lip sync engine - lipSync_engine: LipSyncEngine; + // plays audio + audio_engine: AudioEngine; + // key framing + transform_engine: TransformEngine; + // animation engine + animation_engine: AnimationEngine; + // lip sync engine + lipSync_engine: LipSyncEngine; - scene: Scene; + scene: Scene; - current_time: number; - // ensure that the elements are loaded first. - constructor( - audio_engine: AudioEngine, - transform_engine: TransformEngine, - lipsync_engine: LipSyncEngine, - animation_engine: AnimationEngine, - scene: Scene, - ) { - this.timeline_items = []; - this.absolute_end = 60 * 12; - this.timeline_limit = this.absolute_end; // 5 seconds - - this.is_playing = false; - this.scrubber_frame_position = 0; // in frames into the tl - - // this will be used to play the audio clips - this.audio_engine = audio_engine; - this.transform_engine = transform_engine; - this.lipSync_engine = lipsync_engine; - this.animation_engine = animation_engine; - - this.scene = scene; - - Queue.subscribe( - QueueNames.TO_ENGINE, - this.handleTimelineActions.bind(this), - ); - - this.current_time = 0; - // TODO: How to move the timeline should put in update. - // setInterval(()=> { - // this.current_time +=1 - // this.pushEvent(fromEngineActions.UPDATE_TIME, { currentTime: this.current_time }) - // },50) - } - - public async pushEvent(action: fromEngineActions, data: any) { - this.current_time += 1; - Queue.publish({ - queueName: QueueNames.FROM_ENGINE, - action: fromEngineActions.UPDATE_TIME, - data: data, - }); - } - - public async handleTimelineActions(data: any) { - const action = data["action"]; - switch (action) { - case toEngineActions.ADD_KEYFRAME: - await this.addKeyFrame(data); - break; - case toEngineActions.UPDATE_KEYFRAME: - await this.updateKeyFrame(data); - break; - case toEngineActions.DELETE_KEYFRAME: - await this.deleteKeyFrame(data); - break; - case toEngineActions.ADD_CLIP: - await this.addClip(data); - break; - case toEngineActions.DELETE_CLIP: - await this.deleteClip(data); - break; - case toEngineActions.UPDATE_CLIP: - await this.updateClip(data); - break; - case toEngineActions.UPDATE_TIME: - await this.scrub(data); - break; - case toEngineActions.MUTE: - await this.mute(data); - break; - default: - console.log("Action Not Wired"); - } - } - - public async addKeyFrame(data: any) { - // KeyFrame Object - // version: number; - // clip_uuid: string; - // group: ClipGroup; - // object_uuid?: string; - // offset: number; - // position: XYZ - // rotation: XYZ; - // scale: XYZ; - // selected?: boolean; - const data_json = data["data"]; - const uuid = data_json["object_uuid"]; - - let object_name = this.scene.get_object_by_uuid(uuid)?.name; - if (object_name == undefined) { - object_name = "undefined"; - } - - const new_item = this.transform_engine.addFrame( - uuid, - this.absolute_end, - data_json["position"], - data_json["rotation"], - data_json["scale"], - data_json["offset"], - data_json["keyframe_uuid"], - ); - if (new_item) { - await this.addPlayableClip( - new ClipUI( - data_json["version"], - ClipType.TRANSFORM, - data_json["group"], - object_name, - "", - uuid, - 0, - this.absolute_end, - ), - ); - } - } - - public async addClip(data: any) { - const object_uuid = data["data"]["object_uuid"]; - const media_id = data["data"]["media_id"]; - const name = data["data"]["name"]; - const group = data["data"]["group"]; - const version = data["data"]["group"]; - const type = data["data"]["type"]; - const offset = data["data"]["offset"]; - const end_offset = data["data"]["length"] + offset; - - switch (type) { - case "animation": - this.animation_engine.load_object(object_uuid, media_id, name); - break; - case "transform": - this.transform_engine.loadObject(object_uuid, data["data"]["length"]); - break; - case "audio": - if (group == "character") { - this.lipSync_engine.load_object(object_uuid, media_id); - // media id for this as well it can be downloaded - this.addPlayableClip( - new ClipUI( - version, - ClipType.AUDIO, - group, - name, - media_id, - object_uuid, - offset, - end_offset, - ), - ); - return; - } else { - console.log("Audio!"); - this.audio_engine.loadClip(media_id); - } - break; - } - - // media id for this as well it can be downloaded - this.addPlayableClip( - new ClipUI( - version, - type, - group, - name, - media_id, - object_uuid, - offset, - end_offset, - ), - ); - } - - public async deleteKeyFrame(data: any) { - const keyframe_uuid = data["data"]["keyframe_uuid"]; - const object_uuid = data["data"]["object_uuid"]; - this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid); - } - - public async updateKeyFrame(data: any) { - const keyframe_uuid = data["data"]["keyframe_uuid"]; - const keyframe_offset = data["data"]["offset"]; - const object_uuid = data["data"]["object_uuid"]; - this.transform_engine.clips[object_uuid].setOffset( - keyframe_uuid, - keyframe_offset, - ); - } - - public async updateClip(data: any) { - // only length and offset changes here. - console.log(data); - - const object_uuid = data["data"]["object_uuid"]; - const media_id = data["data"]["media_id"]; - const offset = data["data"]["offset"]; - const length = data["data"]["length"] + offset; - - for (const element of this.timeline_items) { - if (element.media_id == media_id && element.object_uuid == object_uuid) { - element.length = length; - element.offset = offset; - } - } - } - public async deleteClip(data: any) { - console.log(data); - } - - public async scrubberUpdate(data: any) { - console.log(data); - } - - public async mute(data: any) { - console.log(data); - } - - public async addPlayableClip(clip: ClipUI): Promise { - this.timeline_items.push(clip); - } - - // when given a media id item it will create the clip. - // Then the clip will be loaded by the engines, if they come from outside of the loaded scene. - public async createClipOffset( - media_id: string, - object_uuid: string, - type: string, - ): Promise { - // use engine to load based off media id and type animation | transform | - } - - // this will update the state of the clips based off uuid easing? - public async updatePlayableClip( - clip_uuid: string, - updates: AnyJson, - ): Promise {} - - public async deletePlayableClip(clip_uuid: string): Promise {} - - public async scrub(data: any): Promise { - if (this.is_playing) { - return; - } - this.update(); - this.setScrubberPosition(data["data"]["currentTime"]); - } - - public async scrubberDidStop(offset_frame: number) {} - // public streaming events into the timeline from - public async setScrubberPosition(offset: number) { - this.scrubber_frame_position = offset; // in ms - } - - // should play from the clip that is closest to the to scrubber - public async play(): Promise { - console.log(`Starting Timeline`); - this.is_playing = true; - } - - private async resetScene() { - this.timeline_limit = this.getEndPoint(); - for (const element of this.timeline_items) { - if (element.type == "transform") { - const object = this.scene.get_object_by_uuid(element.object_uuid); - if (object && this.transform_engine.clips[element.object_uuid]) { - this.transform_engine.clips[element.object_uuid].reset(object); - } - } else if (element.type == "audio") { - this.audio_engine.loadClip(element.media_id); - } else if (element.type == "animation") { - this.animation_engine.clips[element.object_uuid].stop(); - } else if (element.type == "lipsync") { - this.lipSync_engine.clips[element.object_uuid].reset(); - } else { - this.stop(); - throw "Error New Type of element in the timeline"; - } - } - } - - private getEndPoint(): number { - let longest = 0; - for (const element of this.timeline_items) { - if (longest < element.length) { - longest = element.length; - } - } - return longest; - } - - // called by the editor update loop on each frame - public async update() { - //if (this.is_playing == false) return; // start and stop - - if (this.is_playing) { - this.current_time += 1; - this.pushEvent(fromEngineActions.UPDATE_TIME, { - currentTime: this.current_time, - }); - this.scrubber_frame_position = this.current_time; - } - - if (this.scrubber_frame_position <= 0) { - await this.resetScene(); - } - - //this.scrubber_frame_position += 1; - //2. allow stopping. - //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. - for (const element of this.timeline_items) { - if ( - element.offset <= this.scrubber_frame_position && - this.scrubber_frame_position <= element.length - ) { - // run async - // element.play() - // remove the element from the list - const object = this.scene.get_object_by_uuid(element.object_uuid); - if (element.type === ClipType.TRANSFORM) { - if (object && this.transform_engine.clips[element.object_uuid]) { - this.transform_engine.clips[element.object_uuid].step( - object, - element.offset, - this.scrubber_frame_position, - ); - element.length = - this.transform_engine.clips[element.object_uuid].length; - } - } else if ( - element.group !== ClipGroup.CHARACTER && - element.type === ClipType.AUDIO - ) { - if (this.scrubber_frame_position + 1 >= element.length) { - this.audio_engine.stopClip(element.media_id); - } else { - this.audio_engine.playClip(element.media_id); - } - } else if ( - element.group === ClipGroup.CHARACTER && - element.type === ClipType.AUDIO - ) { - if (this.scrubber_frame_position + 1 >= element.length) { - this.lipSync_engine.clips[element.object_uuid].stop(); - } else if (object) { - await this.lipSync_engine.clips[element.object_uuid].play(object); - this.lipSync_engine.clips[element.object_uuid].step(); - } - } else if (element.type === ClipType.ANIMATION) { - if (object) { - await this.animation_engine.clips[object.uuid].play(object); - this.animation_engine.clips[object.uuid].step( - this.scrubber_frame_position / 120, // Double FPS for best result. - ); - } - } else { - this.stop(); - throw "Error New Type of element in the timeline"; - } - //this.timelineItems = this.timelineItems.filter(item => item !== element) - } - } - - if ( - this.scrubber_frame_position >= this.timeline_limit && - this.is_playing + current_time: number; + // ensure that the elements are loaded first. + constructor( + audio_engine: AudioEngine, + transform_engine: TransformEngine, + lipsync_engine: LipSyncEngine, + animation_engine: AnimationEngine, + scene: Scene, ) { - await this.stop(); - } - } + this.timeline_items = []; + this.absolute_end = 60 * 12; + this.timeline_limit = this.absolute_end; // 5 seconds - private async stop(): Promise { - await this.resetScene(); - this.is_playing = false; - console.log(`Stopping Timeline`); - this.current_time = 0; - this.pushEvent(fromEngineActions.UPDATE_TIME, { - currentTime: this.current_time, - }); - } + this.is_playing = false; + this.scrubber_frame_position = 0; // in frames into the tl + + // this will be used to play the audio clips + this.audio_engine = audio_engine; + this.transform_engine = transform_engine; + this.lipSync_engine = lipsync_engine; + this.animation_engine = animation_engine; + + this.scene = scene; + + Queue.subscribe( + QueueNames.TO_ENGINE, + this.handleTimelineActions.bind(this), + ); + + this.current_time = 0; + // TODO: How to move the timeline should put in update. + // setInterval(()=> { + // this.current_time +=1 + // this.pushEvent(fromEngineActions.UPDATE_TIME, { currentTime: this.current_time }) + // },50) + } + + public async pushEvent(action: fromEngineActions, data: any) { + this.current_time += 1; + Queue.publish({ + queueName: QueueNames.FROM_ENGINE, + action: fromEngineActions.UPDATE_TIME, + data: data, + }); + } + + public async handleTimelineActions(data: any) { + const action = data["action"]; + switch (action) { + case toEngineActions.ADD_KEYFRAME: + await this.addKeyFrame(data); + break; + case toEngineActions.UPDATE_KEYFRAME: + await this.updateKeyFrame(data); + break; + case toEngineActions.DELETE_KEYFRAME: + await this.deleteKeyFrame(data); + break; + case toEngineActions.ADD_CLIP: + await this.addClip(data); + break; + case toEngineActions.DELETE_CLIP: + await this.deleteClip(data); + break; + case toEngineActions.UPDATE_CLIP: + await this.updateClip(data); + break; + case toEngineActions.UPDATE_TIME: + await this.scrub(data); + break; + case toEngineActions.MUTE: + await this.mute(data); + break; + default: + console.log("Action Not Wired"); + } + } + + public async addKeyFrame(data: any) { + // KeyFrame Object + // version: number; + // clip_uuid: string; + // group: ClipGroup; + // object_uuid?: string; + // offset: number; + // position: XYZ + // rotation: XYZ; + // scale: XYZ; + // selected?: boolean; + let data_json = data['data']; + let uuid = data_json['object_uuid']; + + let object_name = this.scene.get_object_by_uuid(uuid)?.name; + if (object_name == undefined) { + object_name = "undefined" + } + + let new_item = this.transform_engine.addFrame(uuid, + this.absolute_end, + data_json['position'], + data_json['rotation'], + data_json['scale'], + data_json['offset'], + data_json['keyframe_uuid']); + if (new_item) { + await this.addPlayableClip(new ClipUI( + data_json['version'], + ClipType.TRANSFORM, + data_json['group'], + object_name, + "", + uuid, + 0, + this.absolute_end)) + } + } + + public async addClip(data: any) { + const object_uuid = data["data"]["object_uuid"]; + const media_id = data["data"]["media_id"]; + const name = data["data"]["name"]; + const group = data["data"]["group"]; + const version = data["data"]["group"]; + const type = data["data"]["type"]; + const offset = data["data"]["offset"]; + const end_offset = data["data"]["length"] + offset; + + switch (type) { + case "animation": + this.animation_engine.load_object(object_uuid, media_id, name); + break; + case "transform": + this.transform_engine.loadObject(object_uuid, data["data"]["length"]); + break; + case "audio": + if (group == "character") { + this.lipSync_engine.load_object(object_uuid, media_id); + // media id for this as well it can be downloaded + this.addPlayableClip( + new ClipUI( + version, + "lipsync",// TODO potiential bug .. + group, + name, + media_id, + object_uuid, + offset, + end_offset)); + return; + } else { + console.log("Audio!") + this.audio_engine.loadClip(media_id); + } + break; + } + + // media id for this as well it can be downloaded + this.addPlayableClip( + new ClipUI( + version, + type, + group, + name, + media_id, + object_uuid, + offset, + end_offset, + ), + ); + } + + public async deleteKeyFrame(data: any) { + let keyframe_uuid = data['data']["keyframe_uuid"]; + let object_uuid = data['data']['object_uuid']; + this.transform_engine.clips[object_uuid].removeKeyframe(keyframe_uuid); + } + + public async updateKeyFrame(data: any) { + let keyframe_uuid = data['data']["keyframe_uuid"]; + let keyframe_offset = data['data']["offset"]; + let object_uuid = data['data']['object_uuid']; + this.transform_engine.clips[object_uuid].setOffset(keyframe_uuid, keyframe_offset); + } + + public async updateClip(data: any) { + // only length and offset changes here. + console.log(data); + + let object_uuid = data["data"]["object_uuid"]; + const media_id = data["data"]["media_id"]; + const offset = data["data"]["offset"]; + const length = data["data"]["length"] + offset; + + for (const element of this.timeline_items) { + if (element.media_id == media_id && element.object_uuid == object_uuid) { + element.length = length; + element.offset = offset; + } + } + } + public async deleteClip(data: any) { + console.log(data); + } + + public async scrubberUpdate(data: any) { + console.log(data); + } + + public async mute(data: any) { + console.log(data); + } + + public async addPlayableClip(clip: ClipUI): Promise { + this.timeline_items.push(clip); + } + + // when given a media id item it will create the clip. + // Then the clip will be loaded by the engines, if they come from outside of the loaded scene. + public async createClipOffset( + media_id: string, + object_uuid: string, + type: string, + ): Promise { + // use engine to load based off media id and type animation | transform | + } + + // this will update the state of the clips based off uuid easing? + public async updatePlayableClip( + clip_uuid: string, + updates: AnyJson, + ): Promise { } + + public async deletePlayableClip(clip_uuid: string): Promise { } + + public async scrub(data: any): Promise { + if (this.is_playing) { + return; + } + this.update(); + this.setScrubberPosition(data["data"]["currentTime"]); + } + + public async scrubberDidStop(offset_frame: number) { } + // public streaming events into the timeline from + public async setScrubberPosition(offset: number) { + this.scrubber_frame_position = offset; // in ms + } + + // should play from the clip that is closest to the to scrubber + public async play(): Promise { + console.log(`Starting Timeline`); + this.is_playing = true; + } + + private async resetScene() { + this.timeline_limit = this.getEndPoint(); + for (const element of this.timeline_items) { + if (element.type == "transform") { + const object = this.scene.get_object_by_uuid(element.object_uuid); + if (object && this.transform_engine.clips[element.object_uuid]) { + this.transform_engine.clips[element.object_uuid].reset(object); + } + } else if (element.type == "audio") { + this.audio_engine.loadClip(element.media_id); + } else if (element.type == "animation") { + this.animation_engine.clips[element.object_uuid].stop(); + } else if (element.type == "lipsync") { + this.lipSync_engine.clips[element.object_uuid].reset(); + } else { + this.stop(); + throw "Error New Type of element in the timeline"; + } + } + } + + private getEndPoint(): number { + let longest = 0; + for (const element of this.timeline_items) { + if (longest < element.length) { + longest = element.length; + } + } + return longest; + } + + // called by the editor update loop on each frame + public async update() { + //if (this.is_playing == false) return; // start and stop + + if (this.is_playing) { + this.current_time += 1; + this.pushEvent(fromEngineActions.UPDATE_TIME, { + currentTime: this.current_time, + }); + this.scrubber_frame_position = this.current_time; + } + + if (this.scrubber_frame_position <= 0) { + await this.resetScene(); + } + + //this.scrubber_frame_position += 1; + //2. allow stopping. + //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. + for (const element of this.timeline_items) { + if ( + element.offset <= this.scrubber_frame_position && + this.scrubber_frame_position <= element.length + ) { + // run async + // element.play() + // remove the element from the list + const object = this.scene.get_object_by_uuid(element.object_uuid); + if (element.type == "transform") { + if (object && this.transform_engine.clips[element.object_uuid]) { + this.transform_engine.clips[element.object_uuid].step( + object, + element.offset, + this.scrubber_frame_position, + ); + element.length = this.transform_engine.clips[element.object_uuid].length; + } + } else if (element.type == "audio") { + if (this.scrubber_frame_position + 1 >= element.length) { + this.audio_engine.stopClip(element.media_id); + } else { + this.audio_engine.playClip(element.media_id); + } + } else if (element.type == "lipsync") { // we will remove this when we know which group it will come from character + audio == lip sync audio. + if (this.scrubber_frame_position + 1 >= element.length) { + this.lipSync_engine.clips[element.object_uuid].stop(); + } else if (object) { + await this.lipSync_engine.clips[element.object_uuid].play(object); + this.lipSync_engine.clips[element.object_uuid].step(); + } + } else if (element.type == "animation") { + if (object) { + await this.animation_engine.clips[object.uuid].play(object); + this.animation_engine.clips[object.uuid].step( + this.scrubber_frame_position / 120, // Double FPS for best result. + ); + } + } else { + this.stop(); + throw "Error New Type of element in the timeline"; + } + //this.timelineItems = this.timelineItems.filter(item => item !== element) + } + } + + if (this.scrubber_frame_position >= this.timeline_limit && this.is_playing) { + await this.stop(); + } + } + + private async stop(): Promise { + await this.resetScene(); + this.is_playing = false; + console.log(`Stopping Timeline`); + this.current_time = 0; + this.pushEvent(fromEngineActions.UPDATE_TIME, { + currentTime: this.current_time, + }); + } } // How much timeline precision we have diff --git a/app/pages/PageEnigma/models/track.ts b/app/pages/PageEnigma/models/track.ts index a6d316ed42..5e46e23102 100644 --- a/app/pages/PageEnigma/models/track.ts +++ b/app/pages/PageEnigma/models/track.ts @@ -38,6 +38,7 @@ export enum ClipType { TRANSFORM = "transform", AUDIO = "audio", ANIMATION = "animation", + LIPSYNC = "lipsync" } export enum ClipGroup { From f956e65bc62e668d3a0aae5f707dbbfcc6c2e6c0 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Thu, 4 Apr 2024 17:12:54 -0400 Subject: [PATCH 5/5] Removed buttons that were no longer needed moved functionality to generate movie cross checking if there are odd keyframing issues. --- app/pages/PageEnigma/PageEnigmaComponent.tsx | 15 +++- .../ControlsTopButtons/ControlsTopButtons.tsx | 78 +------------------ .../contexts/TrackContext/TrackProvider.tsx | 47 +---------- app/pages/PageEnigma/js/timeline.ts | 4 +- 4 files changed, 20 insertions(+), 124 deletions(-) diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index b8416749c1..160c9ca07e 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -1,5 +1,4 @@ import { useContext } from "react"; - import { faSparkles } from "@fortawesome/pro-solid-svg-icons"; import { Button, LoadingBar, LoadingDots } from "~/components"; @@ -19,9 +18,12 @@ import { APPUI_VIEW_MODES } from "./reducers"; import { timelineHeight } from "~/pages/PageEnigma/store"; import { useSignals } from "@preact/signals-react/runtime"; import { AppUiContext } from "~/pages/PageEnigma/contexts/AppUiContext"; +import { EngineContext } from "./contexts/EngineContext"; export const PageEnigmaComponent = () => { useSignals(); + + const editorEngine = useContext(EngineContext); const [appUiState] = useContext(AppUiContext); //To prevent the click event from propagating to the canvas: TODO: HANDLE THIS BETTER? @@ -29,13 +31,20 @@ export const PageEnigmaComponent = () => { event.stopPropagation(); }; + const handleGenerateMovieClick=()=> { + if ( editorEngine != null) { + editorEngine?.generateVideo() + } else { + console.log("Tried to generate movie but editor was null") + } + }; + const lowerHeight = timelineHeight.value; return (
- + -
{/* Engine section/side panel */}
{ const handleButtonTest = () => { editorEngine?.testStylizeRequest(); - - // if(appUiState?.showEditorLoadingBar.isShowing){ - //TO GO TO A HUNDRED AND DISAPPER - // dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR, - // payload:{ - // showEditorLoadingBar: { - // progress: 100, - // } - // } - // }); - // setTimeout(() => { - // dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADINGBAR, - // }); - // }, 1000); - //END :TO GO TO A HUNDRED AND DISAPPER - - //EAMPLE OF CHANING LOADING MESSAGE - // dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR, - // payload:{ - // showEditorLoadingBar: { - // message: "new message", - // } - // } - // }); - // }else{ - // dispatchAppUiState({ - // type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADINGBAR, - // payload:{ - // showEditorLoadingBar: { - // message: 'display of LoadingBar triggered by Test Button', - // progress: 10, - // } - // } - - // }); - //} }; - const handleObjectPanelTest = () => { - if (appUiState?.controlPanel.isShowing) { - dispatchAppUiState({ - type: APPUI_ACTION_TYPES.HIDE_CONTROLPANELS_SCENEOBJECT, - }); - } else { - dispatchAppUiState({ - type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT, - payload: { - group: ClipGroup.OBJECT, - object_name: "TEST BUTTON", - object_uuid: "", - version: "1", - objectVectors: { - position: { x: 1, y: 2, z: 3 }, - rotation: { x: 4, y: 5, z: 6 }, - scale: { x: 7, y: 8, z: 9 }, - }, - }, - }); - } + const handleTestButton2 = () => { + console.log("Test Button 2") }; const handleButtonCameraView = () => { @@ -94,18 +36,9 @@ export const ControlsTopButtons = () => { const handleButtonPlayBack = () => { editorEngine?.startPlayback(); }; - const handleButtonRender = () => { - editorEngine?.generateVideo(); - }; - const handleButtonTakeFrame = () => { editorEngine?.take_timeline_cam_clip(); }; - - const handleButtonLoad = () => { - console.log("LOADING"); - }; - const handleButtonSingleFrame = () => { editorEngine?.generateFrame(); }; @@ -172,18 +105,15 @@ export const ControlsTopButtons = () => { Toggle Camera View
- - {/* - */}