From 0038a40a27f688eb3462405bc72044a11075dab6 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Fri, 29 Mar 2024 20:57:27 +0700 Subject: [PATCH 01/10] Move with scrollbar --- app/pages/PageEnigma/PageEnigmaComponent.tsx | 28 ++++++- .../comps/SidePanelTabs/ClipElement.tsx | 73 ++++++------------- .../comps/SidePanelTabs/ClipElements.tsx | 12 ++- app/pages/PageEnigma/store/dragAndDrop.ts | 8 ++ app/root.tsx | 2 +- 5 files changed, 66 insertions(+), 57 deletions(-) diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index bc02df71f1..7d442cd9d6 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -17,12 +17,21 @@ import { AppUIProvider } from "./contexts/AppUiContext"; import { reducer, initialState, ACTION_TYPES } from "./reducer"; import { VIEW_MODES } from "./reducer/types"; import { ViewSideBySide } from "./comps/ViewSideBySide"; -import { timelineHeight } from "~/pages/PageEnigma/store"; +import { + canDrop, + clipLength, + currPosition, + dragId, + overTimeline, + scale, + timelineHeight, +} from "~/pages/PageEnigma/store"; import { useSignals } from "@preact/signals-react/runtime"; export const PageEnigmaComponent = () => { const [appUiState, dispatchAppUiState] = useReducer(reducer, initialState); useSignals(); + const { currX, currY } = currPosition.value; useEffect(() => { setTimeout( @@ -35,7 +44,6 @@ export const PageEnigmaComponent = () => { }, []); const lowerHeight = timelineHeight.value; - console.log("main", timelineHeight.value); return (
@@ -98,6 +106,22 @@ export const PageEnigmaComponent = () => {
+
); }; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx index 715a956933..45b1523a48 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx @@ -1,7 +1,12 @@ -import { useCallback, useContext, useEffect, useState } from "react"; +import { useCallback, useContext, useEffect } from "react"; import { BaseClip } from "~/pages/PageEnigma/models/track"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; -import { canDrop, dragId, overTimeline, scale } from "~/pages/PageEnigma/store"; +import { + canDrop, + currPosition, + dragId, + initPosition, +} from "~/pages/PageEnigma/store"; import { useSignals } from "@preact/signals-react/runtime"; interface Props { @@ -12,22 +17,7 @@ interface Props { export const ClipElement = ({ clip, type }: Props) => { useSignals(); const { startDrag, endDrag } = useContext(TrackContext); - const [initPosition, setInitPosition] = useState<{ - initX: number; - initY: number; - }>({ - initX: 0, - initY: 0, - }); - const [currPosition, setCurrPosition] = useState<{ - x: number; - y: number; - }>({ - x: 0, - y: 0, - }); - const { x, y } = currPosition; - const { initX, initY } = initPosition; + const { initX, initY } = initPosition.value; useEffect(() => { const onPointerUp = () => { @@ -40,9 +30,9 @@ export const ClipElement = ({ clip, type }: Props) => { if (dragId) { event.stopPropagation(); event.preventDefault(); - const deltaX = event.clientX - initX; - const deltaY = event.clientY - initY; - setCurrPosition({ x: deltaX, y: deltaY }); + const deltaX = event.pageX - initX; + const deltaY = event.pageY - initY; + currPosition.value = { currX: initX + deltaX, currY: initY + deltaY }; return; } }; @@ -59,22 +49,20 @@ export const ClipElement = ({ clip, type }: Props) => { const onPointerDown = useCallback( (event: React.PointerEvent) => { if (event.button === 0) { - const box = document.getElementById(`ani-clip-${clip.id}`); - const position = box!.getBoundingClientRect(); - + console.log("X", event.clientX); startDrag(type, clip.id, clip.length); - setCurrPosition({ - x: position.x, - y: position.y + position.height + 1, - }); - setInitPosition({ - initX: position.x, - initY: position.y + position.height + 1, - }); + currPosition.value = { + currX: event.pageX, + currY: event.pageY, + }; + initPosition.value = { + initX: event.pageX, + initY: event.pageY, + }; canDrop.value = false; } }, - [type, clip.id, startDrag], + [type, clip.id, startDrag, clip.length], ); return ( @@ -87,25 +75,6 @@ export const ClipElement = ({ clip, type }: Props) => { > {clip.name} -
- {clip.name} -
); }; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx index defcb65ac3..da6c78d5af 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx @@ -1,12 +1,20 @@ -import { useContext } from "react"; +import { useContext, useEffect, useState } from "react"; import { ClipElement } from "./ClipElement"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; +import { useSignals } from "@preact/signals-react/runtime"; +import { timelineHeight } from "~/pages/PageEnigma/store"; export const ClipElements = () => { const { animationClips } = useContext(TrackContext); + useSignals(); + const [height, setHeight] = useState(0); + + useEffect(() => { + setHeight(window.outerHeight - timelineHeight.value); + }, []); return ( -
+
{animationClips.map((clip) => { return ; })} diff --git a/app/pages/PageEnigma/store/dragAndDrop.ts b/app/pages/PageEnigma/store/dragAndDrop.ts index e38765ce73..7d0c98a940 100644 --- a/app/pages/PageEnigma/store/dragAndDrop.ts +++ b/app/pages/PageEnigma/store/dragAndDrop.ts @@ -7,3 +7,11 @@ export const overTimeline = signal(false); export const dropId = signal(""); export const dropOffset = signal(0); export const clipLength = signal(0); +export const initPosition = signal<{ initX: number; initY: number }>({ + initX: 0, + initY: 0, +}); +export const currPosition = signal<{ currX: number; currY: number }>({ + currX: 0, + currY: 0, +}); diff --git a/app/root.tsx b/app/root.tsx index a62b912903..e6504470a3 100644 --- a/app/root.tsx +++ b/app/root.tsx @@ -110,4 +110,4 @@ function CompleteTakeoverLoadingScreen({className}:{className:string}){
); -}; \ No newline at end of file +}; From e38f5c8df0564f524587f3ac254c56200f425a4b Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Fri, 29 Mar 2024 23:02:43 +0700 Subject: [PATCH 02/10] Updates to queuing --- app/pages/PageEnigma/Queue/fromEngineActions.ts | 1 - .../PageEnigma/comps/Timeline/utils/useQueueHandler.ts | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/app/pages/PageEnigma/Queue/fromEngineActions.ts b/app/pages/PageEnigma/Queue/fromEngineActions.ts index 58cbeb4a28..0c258e48c9 100644 --- a/app/pages/PageEnigma/Queue/fromEngineActions.ts +++ b/app/pages/PageEnigma/Queue/fromEngineActions.ts @@ -1,4 +1,3 @@ export enum fromEngineActions { FRAME_TICK, // data={currentTime: number} - OBJECT_ADDED, // data={id: string, name: string, ?????} } diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts index bc9b9176d1..3045576275 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts @@ -1,8 +1,7 @@ import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; import { useSignals } from "@preact/signals-react/runtime"; -import { useCallback, useContext, useEffect } from "react"; +import { useCallback, useEffect } from "react"; import { currentTime } from "~/pages/PageEnigma/store"; -import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; @@ -14,15 +13,10 @@ interface Arguments { export function useQueueHandler() { useSignals(); - const { addObject } = useContext(TrackContext); - const handleActions = useCallback(({ action, data }: Arguments) => { switch (action) { case fromEngineActions.FRAME_TICK: - currentTime.value += 1; - break; - case fromEngineActions.OBJECT_ADDED: - addObject(data); + currentTime.value = data.currentTime; break; default: throw new Error(`Unknown action ${action}`); From 92ed9cd26cb446a0cc5e0b82018e81466d5c147d Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Fri, 29 Mar 2024 23:07:08 +0700 Subject: [PATCH 03/10] Fix adding clips - add sort --- .../contexts/TrackContext/utils/useUpdateCharacters.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts index 4f58f67f08..4b86fd310a 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts +++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts @@ -135,7 +135,7 @@ export default function useUpdateCharacters() { animationClips: [ ...character.animationClips, { ...clip, id: uuid.v4(), offset }, - ], + ].sort((clipA, clipB) => clipA.offset - clipB.offset), }; }); }); @@ -175,7 +175,10 @@ export default function useUpdateCharacters() { } return { ...character, - lipSyncClips: [...character.lipSyncClips, { ...clip, offset }], + lipSyncClips: [ + ...character.lipSyncClips, + { ...clip, id: uuid.v4(), offset }, + ].sort((clipA, clipB) => clipA.offset - clipB.offset), }; }); }); From 4ebba7dfadaab91ee023884868efc312f09b4670 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Sat, 30 Mar 2024 01:00:18 +0700 Subject: [PATCH 04/10] Refactor for queuing --- app/pages/PageEnigma/PageEnigmaComponent.tsx | 3 +- app/pages/PageEnigma/Queue/Queue.ts | 16 +- .../PageEnigma/Queue/fromEngineActions.ts | 3 +- app/pages/PageEnigma/Queue/toEngineActions.ts | 18 +- .../comps/SidePanelTabs/ClipElement.tsx | 14 +- .../comps/SidePanelTabs/ClipElements.tsx | 6 +- .../PageEnigma/comps/Timeline/TrackClips.tsx | 6 +- .../Timeline/utils/useMouseEventsAnimation.ts | 2 +- .../Timeline/utils/useMouseEventsClip.ts | 8 +- .../comps/Timeline/utils/useQueueHandler.ts | 11 +- .../contexts/TrackContext/TrackContext.tsx | 14 +- .../contexts/TrackContext/TrackProvider.tsx | 27 ++- .../TrackContext/utils/useUpdateAudio.ts | 100 +++++---- .../TrackContext/utils/useUpdateCamera.ts | 44 ++-- .../TrackContext/utils/useUpdateCharacters.ts | 202 ++++++++++++------ app/pages/PageEnigma/models/track.ts | 63 ++++-- 16 files changed, 349 insertions(+), 188 deletions(-) diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index 0c94744c75..066ec36412 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -1,4 +1,4 @@ -import { useReducer, useEffect } from "react"; +import { useReducer } from "react"; import { faSparkles } from "@fortawesome/pro-solid-svg-icons"; @@ -17,7 +17,6 @@ import { AppUIProvider } from "./contexts/AppUiContext"; import { appUiReducer, appUiInitialStateValues, - APPUI_ACTION_TYPES, APPUI_VIEW_MODES, } from "./reducers"; import { ViewSideBySide } from "./comps/ViewSideBySide"; diff --git a/app/pages/PageEnigma/Queue/Queue.ts b/app/pages/PageEnigma/Queue/Queue.ts index 5bd8661e64..1bf90a922b 100644 --- a/app/pages/PageEnigma/Queue/Queue.ts +++ b/app/pages/PageEnigma/Queue/Queue.ts @@ -1,14 +1,21 @@ import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; +import { QueueClip } from "~/pages/PageEnigma/models/track"; class Queue { private _queue: Record< string, - { action: fromEngineActions | toEngineActions; data: any }[] + { + action: fromEngineActions | toEngineActions; + data: QueueClip | { currentTime: number }; + }[] > = {}; private _subscribers: Record< string, - (entry: { action: fromEngineActions | toEngineActions; data: any }) => void + (entry: { + action: fromEngineActions | toEngineActions; + data: QueueClip | { currentTime: number }; + }) => void > = {}; public publish({ @@ -18,12 +25,13 @@ class Queue { }: { queueName: string; action: fromEngineActions | toEngineActions; - data: any; + data: QueueClip | { currentTime: number }; }) { if (!this._queue[queueName]) { this._queue[queueName] = []; } this._queue[queueName].push({ action, data }); + console.log("Queued", queueName, action, data); if (this._subscribers[queueName]) { this._subscribers[queueName](this._queue[queueName].splice(0, 1)[0]); @@ -34,7 +42,7 @@ class Queue { queueName: string, onMessage: (entry: { action: fromEngineActions | toEngineActions; - data: any; + data: QueueClip | { currentTime: number }; }) => void, ) { this._subscribers[queueName] = onMessage; diff --git a/app/pages/PageEnigma/Queue/fromEngineActions.ts b/app/pages/PageEnigma/Queue/fromEngineActions.ts index 0c258e48c9..40753b789d 100644 --- a/app/pages/PageEnigma/Queue/fromEngineActions.ts +++ b/app/pages/PageEnigma/Queue/fromEngineActions.ts @@ -1,3 +1,4 @@ export enum fromEngineActions { - FRAME_TICK, // data={currentTime: number} + UPDATE_TIME = "update_time", + // data={currentTime: number} } diff --git a/app/pages/PageEnigma/Queue/toEngineActions.ts b/app/pages/PageEnigma/Queue/toEngineActions.ts index a65a962483..867730e006 100644 --- a/app/pages/PageEnigma/Queue/toEngineActions.ts +++ b/app/pages/PageEnigma/Queue/toEngineActions.ts @@ -1,4 +1,18 @@ export enum toEngineActions { - ADD_ANIMATION, // data={id: string, offset: number, length: number} - SCRUBBER_TIME, // data={currentTime: number} + ADD_CLIP = "add_clip", + // data=QueueClip + DELETE_CLIP = "delete_clip", + // data=QueueClip + MUTE = "mute", + // data={version, type, group, object_uuid} if lipsync + // data={version, type, group} if global_audio + PLAY_CLIP = "play_clip", + // data=QueueClip + UNMUTE = "unmute", + // data={version, type, group, object_uuid} if lipsync + // data={version, type, group} if global_audio + UPDATE_CLIP = "update_clip", + // data=QueueClip + UPDATE_TIME = "update_time", + // data={currentTime: number} } diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx index 45b1523a48..51b0aa1843 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx @@ -1,5 +1,5 @@ import { useCallback, useContext, useEffect } from "react"; -import { BaseClip } from "~/pages/PageEnigma/models/track"; +import { MediaClip } from "~/pages/PageEnigma/models/track"; import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; import { canDrop, @@ -10,7 +10,7 @@ import { import { useSignals } from "@preact/signals-react/runtime"; interface Props { - clip: BaseClip; + clip: MediaClip; type: "animations" | "lipSync"; } @@ -50,7 +50,7 @@ export const ClipElement = ({ clip, type }: Props) => { (event: React.PointerEvent) => { if (event.button === 0) { console.log("X", event.clientX); - startDrag(type, clip.id, clip.length); + startDrag(type, clip.media_id, clip.length); currPosition.value = { currX: event.pageX, currY: event.pageY, @@ -62,14 +62,14 @@ export const ClipElement = ({ clip, type }: Props) => { canDrop.value = false; } }, - [type, clip.id, startDrag, clip.length], + [type, clip.media_id, startDrag, clip.length], ); return ( -
+
diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx index da6c78d5af..1c935c9fc2 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx @@ -14,9 +14,11 @@ export const ClipElements = () => { }, []); return ( -
+
{animationClips.map((clip) => { - return ; + return ( + + ); })}
); diff --git a/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx b/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx index fc37f21df9..f07ee1f593 100644 --- a/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx +++ b/app/pages/PageEnigma/comps/Timeline/TrackClips.tsx @@ -1,7 +1,7 @@ import { TrackClip } from "~/pages/PageEnigma/comps/Timeline/TrackClip"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faVolume, faVolumeSlash } from "@fortawesome/pro-solid-svg-icons"; -import { BaseClip } from "~/pages/PageEnigma/models/track"; +import { Clip } from "~/pages/PageEnigma/models/track"; import { PointerEvent } from "react"; import { canDrop, @@ -15,7 +15,7 @@ import { interface Props { id: string; - clips: BaseClip[]; + clips: Clip[]; title: string; style: "character" | "audio" | "camera" | "objects"; type?: "animations" | "positions" | "lipSync"; @@ -94,7 +94,7 @@ export const TrackClips = ({ > {clips.map((clip, index) => ( 0 ? clips[index - 1].offset + clips[index - 1].length : 0 } diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts index e145579bb9..25de508f55 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts @@ -20,7 +20,7 @@ export const useMouseEventsAnimation = () => { setTime(-1); Queue.publish({ queueName: QueueNames.TO_ENGINE, - action: toEngineActions.SCRUBBER_TIME, + action: toEngineActions.UPDATE_TIME, data: { currentTime: Math.round(time) }, }); } diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts index 8d666599a2..177b392901 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts @@ -1,10 +1,10 @@ import React, { Dispatch, useCallback, useEffect, useRef } from "react"; -import { BaseClip } from "~/pages/PageEnigma/models/track"; +import { Clip } from "~/pages/PageEnigma/models/track"; import { canDrop, scale } from "~/pages/PageEnigma/store"; import { useSignals } from "@preact/signals-react/runtime"; export const useMouseEventsClip = ( - clip: BaseClip, + clip: Clip, max: number, min: number, updateClip: (args: { id: string; offset: number; length: number }) => void, @@ -21,14 +21,14 @@ export const useMouseEventsClip = ( const onPointerUp = useCallback(() => { if (isActive.current) { updateClip({ - id: clip.id, + id: clip.clip_uuid, offset: Math.round(currOffset.current), length: Math.round(currLength.current), }); isActive.current = ""; canDrop.value = false; } - }, [updateClip, clip.id]); + }, [updateClip, clip.clip_uuid]); const onMouseMove = useCallback( (event: MouseEvent) => { diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts index 3045576275..3359ce8931 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts @@ -5,18 +5,23 @@ import { currentTime } from "~/pages/PageEnigma/store"; import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; +import { QueueClip } from "~/pages/PageEnigma/models/track"; + +interface UpdateTime { + currentTime: number; +} interface Arguments { action: fromEngineActions | toEngineActions; - data: any; + data: QueueClip | UpdateTime; } export function useQueueHandler() { useSignals(); const handleActions = useCallback(({ action, data }: Arguments) => { switch (action) { - case fromEngineActions.FRAME_TICK: - currentTime.value = data.currentTime; + case fromEngineActions.UPDATE_TIME: + currentTime.value = (data as UpdateTime).currentTime; break; default: throw new Error(`Unknown action ${action}`); diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx index bd523c4041..f761df637e 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx @@ -1,13 +1,11 @@ import { createContext } from "react"; import { - AnimationClip, - AudioClip, AudioGroup, - BaseClip, CameraGroup, CharacterGroup, ObjectGroup, ObjectTrack, + MediaClip, } from "~/pages/PageEnigma/models/track"; export const TrackContext = createContext<{ @@ -23,13 +21,13 @@ export const TrackContext = createContext<{ addCharacterAnimation: (options: { clipId: string; characterId: string; - animationClips: BaseClip[]; + animationClips: MediaClip[]; offset: number; }) => void; addCharacterAudio: (options: { clipId: string; characterId: string; - audioClips: BaseClip[]; + audioClips: MediaClip[]; offset: number; }) => void; selectCharacterClip: (clipId: string) => void; @@ -55,7 +53,7 @@ export const TrackContext = createContext<{ toggleAudioMute: () => void; addGlobalAudio: ( clipId: string, - animationClips: BaseClip[], + animationClips: MediaClip[], offset: number, ) => void; selectAudioClip: (clipId: string) => void; @@ -71,8 +69,8 @@ export const TrackContext = createContext<{ selectClip: (clipId: string) => void; // sidebar clips - animationClips: AnimationClip[]; - audioClips: AudioClip[]; + animationClips: MediaClip[]; + audioClips: MediaClip[]; // drag and drop startDrag: ( diff --git a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx index 413b947e99..bc528311bd 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx +++ b/app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx @@ -4,7 +4,7 @@ import useUpdateCharacters from "~/pages/PageEnigma/contexts/TrackContext/utils/ import useUpdateCamera from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera"; import useUpdateAudio from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio"; import useUpdateObject from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateObject"; -import { AnimationClip, AudioClip } from "~/pages/PageEnigma/models/track"; +import { ClipType, MediaClip } from "~/pages/PageEnigma/models/track"; import useUpdateDragDrop from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop"; import { canDrop, @@ -13,6 +13,7 @@ import { dropId, dropOffset, } from "~/pages/PageEnigma/store"; +import * as uuid from "uuid"; interface Props { children: ReactNode; @@ -31,8 +32,8 @@ export const TrackProvider = ({ children }: Props) => { setSelectedClip(clipId); }, []); - const [animationClips, setAnimationClips] = useState([]); - const [audioClips, setAudioClips] = useState([]); + const [animationClips, setAnimationClips] = useState([]); + const [audioClips, setAudioClips] = useState([]); const [scale, setScale] = useState(1); const [length, setLength] = useState(1); @@ -74,26 +75,30 @@ export const TrackProvider = ({ children }: Props) => { useEffect(() => { setAnimationClips([ { - id: "ani-id1", - offset: 0, + version: 1, + media_id: uuid.v4(), + type: ClipType.ANIMATION, length: 25, name: "Sit", }, { - id: "ani-id2", - offset: 0, + version: 1, + media_id: uuid.v4(), + type: ClipType.ANIMATION, length: 25, name: "Idle", }, { - id: "ani-id3", - offset: 0, + version: 1, + media_id: uuid.v4(), + type: ClipType.ANIMATION, length: 25, name: "Stand", }, { - id: "ani-id4", - offset: 0, + version: 1, + media_id: uuid.v4(), + type: ClipType.ANIMATION, length: 25, name: "Walk", }, diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts index 2052926c16..5804527373 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts +++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts @@ -1,24 +1,21 @@ import { useCallback, useState } from "react"; -import { AudioGroup, BaseClip } from "~/pages/PageEnigma/models/track"; +import { + AudioGroup, + Clip, + ClipGroup, + ClipType, + MediaClip, +} from "~/pages/PageEnigma/models/track"; +import Queue from "~/pages/PageEnigma/Queue/Queue"; +import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; +import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; +import * as uuid from "uuid"; export default function useUpdateAudio() { const [audio, setAudio] = useState({ id: "AU1", muted: false, - clips: [ - { - id: "AU1-1", - length: 200, - offset: 0, - name: "audio 1", - }, - { - id: "AU1-2", - length: 180, - offset: 300, - name: "audio 2", - }, - ], + clips: [], }); const updateAudio = useCallback( @@ -33,44 +30,56 @@ export default function useUpdateAudio() { }) => { setAudio((oldAudio) => { const newClips = [...oldAudio.clips]; - const clipIndex = newClips.findIndex((row) => row.id === id); + const clipIndex = newClips.findIndex((row) => row.clip_uuid === id); if (clipIndex === -1) { return { ...oldAudio }; } const clip = newClips[clipIndex]; clip.offset = offset; clip.length = length; + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.UPDATE_CLIP, + data: clip, + }); + return { ...oldAudio, clips: newClips, }; }); - console.log("message", { - action: "UpdateGlobalAudio", - id, - data: { offset, length }, - }); }, [], ); const addGlobalAudio = useCallback( - (dragId: string, audioClips: BaseClip[], offset: number) => { - const clip = audioClips.find((row) => row.id === dragId); + (dragId: string, audioClips: MediaClip[], offset: number) => { + const clip = audioClips.find((row) => row.media_id === dragId); if (!clip) { return; } + const clip_uuid = uuid.v4(); + const newClip = { + ...clip, + type: ClipType.AUDIO, + group: ClipGroup.GLOBAL_AUDIO, + offset, + clip_uuid, + } as Clip; + setAudio((oldAudio) => { return { ...oldAudio, - clips: [...oldAudio.clips, { ...clip, offset }], + clips: [...oldAudio.clips, newClip], }; }); - console.log("message", { - action: "UpdateGlobalAudio", - id: dragId, - data: { offset }, + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.ADD_CLIP, + data: newClip, }); }, [], @@ -78,15 +87,21 @@ export default function useUpdateAudio() { const toggleAudioMute = useCallback(() => { setAudio((oldAudio) => { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: oldAudio?.muted ? toEngineActions.UNMUTE : toEngineActions.MUTE, + data: { + version: 1, + type: ClipType.AUDIO, + group: ClipGroup.GLOBAL_AUDIO, + }, + }); + return { ...oldAudio, muted: !oldAudio.muted, }; }); - console.log("message", { - action: "ToggleGlobalAudioMute", - id: "", - }); }, []); const selectAudioClip = useCallback((clipId: string) => { @@ -97,7 +112,8 @@ export default function useUpdateAudio() { ...oldAudio.clips.map((clip) => { return { ...clip, - selected: clip.id === clipId ? !clip.selected : clip.selected, + selected: + clip.clip_uuid === clipId ? !clip.selected : clip.selected, }; }), ], @@ -109,13 +125,21 @@ export default function useUpdateAudio() { setAudio((oldAudio) => { return { ...oldAudio, - clips: [...oldAudio.clips.filter((clip) => clip.id !== clipId)], + clips: [ + ...oldAudio.clips.filter((clip) => { + if (clip.clip_uuid === clipId) { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.DELETE_CLIP, + data: clip!, + }); + return false; + } + return true; + }), + ], }; }); - console.log("message", { - action: "UpdateGlobalAudio", - id: clipId, - }); }, []); return { diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts index b5b7d21da6..d7bc1395b8 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts +++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts @@ -1,23 +1,13 @@ import { useCallback, useState } from "react"; import { CameraGroup } from "~/pages/PageEnigma/models/track"; +import Queue from "~/pages/PageEnigma/Queue/Queue"; +import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; +import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; export default function useUpdateCamera() { const [camera, setCamera] = useState({ id: "CA1", - clips: [ - { - id: "CA1-1", - length: 200, - offset: 0, - name: "cam 1", - }, - { - id: "CA1-2", - length: 180, - offset: 300, - name: "cam 2", - }, - ], + clips: [], }); const updateCamera = useCallback( @@ -32,13 +22,20 @@ export default function useUpdateCamera() { }) => { setCamera((oldCamera) => { const newClips = [...oldCamera.clips]; - const clipIndex = newClips.findIndex((row) => row.id === id); + const clipIndex = newClips.findIndex((row) => row.clip_uuid === id); if (clipIndex === -1) { return { ...oldCamera }; } const clip = newClips[clipIndex]; clip.offset = offset; clip.length = length; + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.UPDATE_CLIP, + data: clip, + }); + return { ...oldCamera, clips: newClips, @@ -56,7 +53,8 @@ export default function useUpdateCamera() { ...oldCamera.clips.map((clip) => { return { ...clip, - selected: clip.id === clipId ? !clip.selected : clip.selected, + selected: + clip.clip_uuid === clipId ? !clip.selected : clip.selected, }; }), ], @@ -68,7 +66,19 @@ export default function useUpdateCamera() { setCamera((oldCamera) => { return { ...oldCamera, - clips: [...oldCamera.clips.filter((clip) => clip.id !== clipId)], + clips: [ + ...oldCamera.clips.filter((clip) => { + if (clip.clip_uuid === clipId) { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.DELETE_CLIP, + data: clip!, + }); + return false; + } + return true; + }), + ], }; }); }, []); diff --git a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts index 4b86fd310a..cd429c0024 100644 --- a/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts +++ b/app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts @@ -1,5 +1,11 @@ import { useCallback, useState } from "react"; -import { BaseClip, CharacterGroup } from "~/pages/PageEnigma/models/track"; +import { + Clip, + CharacterGroup, + ClipGroup, + ClipType, + MediaClip, +} from "~/pages/PageEnigma/models/track"; import * as uuid from "uuid"; import Queue from "~/pages/PageEnigma/Queue/Queue"; import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; @@ -12,16 +18,10 @@ export default function useUpdateCharacters() { muted: false, animationClips: [], positionClips: [], - lipSyncClips: [ - { - id: "CH1-L1", - length: 400, - offset: 150, - name: "sync 1", - }, - ], + lipSyncClips: [], }, ]); + const updateCharacters = useCallback( ({ type, @@ -39,7 +39,7 @@ export default function useUpdateCharacters() { return oldCharacters.map((character) => { const newAnimationClips = [...character.animationClips]; const clipIndex = newAnimationClips.findIndex( - (row) => row.id === id, + (row) => row.clip_uuid === id, ); if (clipIndex === -1) { return { ...character }; @@ -47,25 +47,28 @@ export default function useUpdateCharacters() { const clip = newAnimationClips[clipIndex]; clip.offset = offset; clip.length = length; + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.UPDATE_CLIP, + data: clip, + }); + return { ...character, animationClips: newAnimationClips, }; }); }); - console.log("message", { - action: "UpdateAnimation", - id, - data: { offset, length }, - }); return; } + if (type === "positions") { setCharacters((oldCharacters) => { return oldCharacters.map((character) => { const newPositionClips = [...character.positionClips]; const clipIndex = newPositionClips.findIndex( - (row) => row.id === id, + (row) => row.clip_uuid === id, ); if (clipIndex === -1) { return { ...character }; @@ -73,6 +76,13 @@ export default function useUpdateCharacters() { const clip = newPositionClips[clipIndex]; clip.offset = offset; clip.length = length; + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.UPDATE_CLIP, + data: clip, + }); + return { ...character, positionClips: newPositionClips, @@ -85,24 +95,28 @@ export default function useUpdateCharacters() { setCharacters((oldCharacters) => { return oldCharacters.map((character) => { const newLipSyncClips = [...character.lipSyncClips]; - const clipIndex = newLipSyncClips.findIndex((row) => row.id === id); + const clipIndex = newLipSyncClips.findIndex( + (row) => row.clip_uuid === id, + ); if (clipIndex === -1) { return { ...character }; } const clip = newLipSyncClips[clipIndex]; clip.offset = offset; clip.length = length; + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.UPDATE_CLIP, + data: clip, + }); + return { ...character, lipSyncClips: newLipSyncClips, }; }); }); - console.log("message", { - action: "UpdateLipSync", - id, - data: { offset, length }, - }); } }, [], @@ -117,14 +131,23 @@ export default function useUpdateCharacters() { }: { clipId: string; characterId: string; - animationClips: BaseClip[]; + animationClips: MediaClip[]; offset: number; }) => { - const clip = animationClips.find((row) => row.id === clipId); + const clip = animationClips.find((row) => row.media_id === clipId); if (!clip) { return; } + const clip_uuid = uuid.v4(); + const newClip = { + ...clip, + group: ClipGroup.CHARACTER, + offset, + clip_uuid, + object_uuid: characterId, + } as Clip; + setCharacters((oldCharacters) => { return oldCharacters.map((character) => { if (character.id !== characterId) { @@ -132,20 +155,17 @@ export default function useUpdateCharacters() { } return { ...character, - animationClips: [ - ...character.animationClips, - { ...clip, id: uuid.v4(), offset }, - ].sort((clipA, clipB) => clipA.offset - clipB.offset), + animationClips: [...character.animationClips, newClip].sort( + (clipA, clipB) => clipA.offset - clipB.offset, + ), }; }); }); + Queue.publish({ queueName: QueueNames.TO_ENGINE, - action: toEngineActions.ADD_ANIMATION, - data: { - clipId, - offset, - }, + action: toEngineActions.ADD_CLIP, + data: newClip, }); }, [], @@ -160,14 +180,24 @@ export default function useUpdateCharacters() { }: { clipId: string; characterId: string; - audioClips: BaseClip[]; + audioClips: MediaClip[]; offset: number; }) => { - const clip = audioClips.find((row) => row.id === clipId); + const clip = audioClips.find((row) => row.media_id === clipId); if (!clip) { return; } + const clip_uuid = uuid.v4(); + const newClip = { + ...clip, + type: ClipType.AUDIO, + group: ClipGroup.CHARACTER, + offset, + clip_uuid, + object_uuid: characterId, + } as Clip; + setCharacters((oldCharacters) => { return oldCharacters.map((character) => { if (character.id !== characterId) { @@ -175,17 +205,17 @@ export default function useUpdateCharacters() { } return { ...character, - lipSyncClips: [ - ...character.lipSyncClips, - { ...clip, id: uuid.v4(), offset }, - ].sort((clipA, clipB) => clipA.offset - clipB.offset), + lipSyncClips: [...character.lipSyncClips, newClip].sort( + (clipA, clipB) => clipA.offset - clipB.offset, + ), }; }); }); - console.log("message", { - action: "AddLipSync", - id: clipId, - data: { offset }, + + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.ADD_CLIP, + data: newClip, }); }, [], @@ -193,15 +223,28 @@ export default function useUpdateCharacters() { const toggleLipSyncMute = useCallback((characterId: string) => { setCharacters((oldCharacters) => { - return oldCharacters.map((character) => ({ - ...character, - muted: - character.id === characterId ? !character.muted : character.muted, - })); - }); - console.log("message", { - action: "ToggleLipSync", - id: characterId, + return oldCharacters.map((character) => { + if (character.id === characterId) { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: character?.muted + ? toEngineActions.UNMUTE + : toEngineActions.MUTE, + data: { + version: 1, + type: ClipType.AUDIO, + group: ClipGroup.CHARACTER, + object_uuid: characterId, + }, + }); + } + + return { + ...character, + muted: + character.id === characterId ? !character.muted : character.muted, + }; + }); }); }, []); @@ -212,15 +255,18 @@ export default function useUpdateCharacters() { ...character, animationClips: character.animationClips.map((clip) => ({ ...clip, - selected: clip.id === clipId ? !clip.selected : clip.selected, + selected: + clip.clip_uuid === clipId ? !clip.selected : clip.selected, })), positionClips: character.positionClips.map((clip) => ({ ...clip, - selected: clip.id === clipId ? !clip.selected : clip.selected, + selected: + clip.clip_uuid === clipId ? !clip.selected : clip.selected, })), lipSyncClips: character.lipSyncClips.map((clip) => ({ ...clip, - selected: clip.id === clipId ? !clip.selected : clip.selected, + selected: + clip.clip_uuid === clipId ? !clip.selected : clip.selected, })), })), ]; @@ -232,22 +278,42 @@ export default function useUpdateCharacters() { return [ ...oldCharacters.map((character) => ({ ...character, - animationClips: character.animationClips.filter( - (clip) => clip.id !== clipId, - ), - positionClips: character.positionClips.filter( - (clip) => clip.id !== clipId, - ), - lipSyncClips: character.lipSyncClips.filter( - (clip) => clip.id !== clipId, - ), + animationClips: character.animationClips.filter((clip) => { + if (clip.clip_uuid === clipId) { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.DELETE_CLIP, + data: clip!, + }); + return false; + } + return true; + }), + positionClips: character.positionClips.filter((clip) => { + if (clip.clip_uuid === clipId) { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.DELETE_CLIP, + data: clip!, + }); + return false; + } + return true; + }), + lipSyncClips: character.lipSyncClips.filter((clip) => { + if (clip.clip_uuid === clipId) { + Queue.publish({ + queueName: QueueNames.TO_ENGINE, + action: toEngineActions.DELETE_CLIP, + data: clip!, + }); + return false; + } + return true; + }), })), ]; }); - console.log("message", { - action: "DeleteCharacterClip", - id: clipId, - }); }, []); return { diff --git a/app/pages/PageEnigma/models/track.ts b/app/pages/PageEnigma/models/track.ts index 32299f513e..dd290db8c6 100644 --- a/app/pages/PageEnigma/models/track.ts +++ b/app/pages/PageEnigma/models/track.ts @@ -1,11 +1,50 @@ -export interface BaseClip { - id: string; +export interface QueueClip { + version: number; + type: ClipType; + group: ClipGroup; + object_uuid?: string; + clip_uuid?: string; + media_id?: string; + name?: string; + offset?: number; + length?: number; + selected?: boolean; +} + +export interface MediaClip { + version: number; + type: ClipType; + media_id: string; + name: string; + length: number; +} + +export interface Clip { + version: number; + clip_uuid: string; + type: ClipType; + group: ClipGroup; + media_id: string; + object_uuid?: string; name: string; offset: number; length: number; selected?: boolean; } +export enum ClipType { + TRANSFORM = "transform", + AUDIO = "audio", + ANIMATION = "animation", +} + +export enum ClipGroup { + CHARACTER = "character", + CAMERA = "camera", + GLOBAL_AUDIO = "global_audio", + OBJECT = "object", +} + export interface BaseKeyFrame { id: string; name: string; @@ -13,35 +52,25 @@ export interface BaseKeyFrame { selected?: boolean; } -export interface AnimationClip extends BaseClip {} - -export interface PositionClip extends BaseClip {} - -export interface LipSyncClip extends BaseClip {} - export interface CharacterGroup { id: string; muted: boolean; - animationClips: AnimationClip[]; - positionClips: PositionClip[]; - lipSyncClips: LipSyncClip[]; + animationClips: Clip[]; + positionClips: Clip[]; + lipSyncClips: Clip[]; } export interface CameraGroup { id: string; - clips: CameraClip[]; + clips: Clip[]; } -export interface CameraClip extends BaseClip {} - export interface AudioGroup { id: string; muted: boolean; - clips: AudioClip[]; + clips: Clip[]; } -export interface AudioClip extends BaseClip {} - export interface ObjectGroup { id: string; objects: ObjectTrack[]; From 6d35e0bad34672aad9b3ce78b280525e6b0a4231 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Sat, 30 Mar 2024 01:41:22 +0700 Subject: [PATCH 05/10] Fix snagging bug --- app/pages/PageEnigma/PageEnigmaComponent.tsx | 1 - .../PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/pages/PageEnigma/PageEnigmaComponent.tsx b/app/pages/PageEnigma/PageEnigmaComponent.tsx index 066ec36412..30b68687e8 100644 --- a/app/pages/PageEnigma/PageEnigmaComponent.tsx +++ b/app/pages/PageEnigma/PageEnigmaComponent.tsx @@ -39,7 +39,6 @@ export const PageEnigmaComponent = () => { appUiReducer, appUiInitialStateValues, ); - useSignals(); const lowerHeight = timelineHeight.value; return ( diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts index 177b392901..be3275ea5e 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts @@ -35,8 +35,11 @@ export const useMouseEventsClip = ( const delta = (event.clientX - clientX.current) / 4 / scale.value; const deltaOffset = delta + initOffset.current; if (isActive.current === "drag") { - if (deltaOffset < min || deltaOffset + currLength.current > max) { - return; + if (deltaOffset < min) { + currOffset.current = min; + } + if (deltaOffset + currLength.current > max) { + currOffset.current = max - currLength.current; } currOffset.current = deltaOffset; } From 9a3c876e3a20ee5704c3d5ab05569b1976b136bc Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Sat, 30 Mar 2024 01:42:47 +0700 Subject: [PATCH 06/10] Fix it again --- app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts index be3275ea5e..1dbe7ed124 100644 --- a/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts +++ b/app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts @@ -37,9 +37,11 @@ export const useMouseEventsClip = ( if (isActive.current === "drag") { if (deltaOffset < min) { currOffset.current = min; + return; } if (deltaOffset + currLength.current > max) { currOffset.current = max - currLength.current; + return; } currOffset.current = deltaOffset; } From 15a0be1c21d720d553374cd027acdc97c0117781 Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Sat, 30 Mar 2024 01:45:58 +0700 Subject: [PATCH 07/10] remove console.log --- app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx index 51b0aa1843..37c09ae69b 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx @@ -49,7 +49,6 @@ export const ClipElement = ({ clip, type }: Props) => { const onPointerDown = useCallback( (event: React.PointerEvent) => { if (event.button === 0) { - console.log("X", event.clientX); startDrag(type, clip.media_id, clip.length); currPosition.value = { currX: event.pageX, From 618041fa01f656eadd244b0e07c2d8adee04524b Mon Sep 17 00:00:00 2001 From: FromTheLOLAshes Date: Sat, 30 Mar 2024 02:12:00 +0700 Subject: [PATCH 08/10] Widen scrubbing bar --- app/pages/PageEnigma/comps/Timeline/Timeline.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx index 4c61defb6c..3fc61d7cad 100644 --- a/app/pages/PageEnigma/comps/Timeline/Timeline.tsx +++ b/app/pages/PageEnigma/comps/Timeline/Timeline.tsx @@ -21,7 +21,7 @@ import { EngineContext } from "../../contexts/EngineContext"; export const Timeline = () => { const editorEngine = useContext(EngineContext); - + const { characters, objects, @@ -148,7 +148,7 @@ export const Timeline = () => { style={{ left: 0, top: 8, - width: 2, + width: 4, height: fullHeight, }} /> From 108e971a560628c0a2a0f3580c4941d876d4695f Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:06:14 -0400 Subject: [PATCH 09/10] Clip TS updated --- .../datastructures/clips/clip_offset.ts | 41 ------------------- app/pages/PageEnigma/js/editor.ts | 6 +-- app/pages/PageEnigma/js/timeline.ts | 12 +++--- 3 files changed, 9 insertions(+), 50 deletions(-) delete mode 100644 app/pages/PageEnigma/datastructures/clips/clip_offset.ts diff --git a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts b/app/pages/PageEnigma/datastructures/clips/clip_offset.ts deleted file mode 100644 index 154c630561..0000000000 --- a/app/pages/PageEnigma/datastructures/clips/clip_offset.ts +++ /dev/null @@ -1,41 +0,0 @@ -// Clip offsets represent the state of the clip on the timeline as well as what type of clip it is. -// it is created from a media id. -export class ClipUI { - version: number - type: "transform" | "audio" | "animation" | "lipsync" - name: string - media_id: string - object_uuid: string - start_offset: number // in frames - ending_offset: number // in frames - - constructor( - version: number, - type: "transform" | "audio" | "animation" | "lipsync", - name: string, - media_id: string, - object_uuid: string, - start_offset: number, - ending_offset: number, - ) { - this.version = version - this.name = name - this.type = type - this.object_uuid = object_uuid - this.media_id = media_id - this.start_offset = start_offset - this.ending_offset = ending_offset - } - - toJSON(): string { - return JSON.stringify({ - version: this.version, - name: this.name, - type: this.type, - object_uuid: this.object_uuid, - media_id: this.media_id, - start_offset: this.start_offset, - ending_offset: this.ending_offset - }) - } -} diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index 0a3c768ac8..e5587d0422 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -16,7 +16,7 @@ import { FFmpeg, createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg"; import AudioEngine from "./audio_engine.js"; import TransformEngine from "./transform_engine.js"; import { TimeLine, TimelineDataState } from "./timeline.js"; -import { ClipUI } from "../datastructures/clips/clip_offset.js"; +import { ClipUI } from "../datastructures/clips/clip_ui.js"; import { LipSyncEngine } from "./lip_sync_engine.js"; import { AnimationEngine } from "./animation_engine.js"; @@ -810,8 +810,8 @@ class Editor { let video_og = itteration + 'tmp.mp4'; let wav_name = itteration + 'tmp.wav'; let new_video = (itteration + 1) + 'tmp.mp4'; - let startFrame = clip.start_offset; - let endFrame = clip.ending_offset; + let startFrame = clip.offset; + let endFrame = clip.length; if (endFrame > this.timeline.timeLineLimit) { diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 267d23ba63..143d4e25a4 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -1,5 +1,5 @@ import { AnyJson } from "three/examples/jsm/nodes/core/constants.js"; -import { ClipUI } from "../datastructures/clips/clip_offset"; +import { ClipUI } from "../datastructures/clips/clip_ui"; import Scene from "./scene.js"; import AudioEngine from "./audio_engine"; @@ -172,19 +172,19 @@ export class TimeLine { //2. allow stopping. //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. for (const element of this.timelineItems) { - if (element.start_offset <= this.scrubberPosition && this.scrubberPosition <= element.ending_offset) { + if (element.offset <= this.scrubberPosition && this.scrubberPosition <= element.length) { // run async // element.play() // remove the element from the list let object = this.scene.get_object_by_uuid(element.object_uuid) if (element.type == "transform") { if (object && this.transformEngine.clips[element.object_uuid]) { - this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset); - this.transformEngine.clips[element.object_uuid].step(object, element.start_offset, this.scrubberPosition); + this.transformEngine.clips[element.object_uuid].length = (element.length - element.offset); + this.transformEngine.clips[element.object_uuid].step(object, element.offset, this.scrubberPosition); } } else if (element.type == "audio") { - if(this.scrubberPosition+1 >= element.ending_offset){ + if(this.scrubberPosition+1 >= element.length){ this.audioEngine.stopClip(element.media_id); } else { @@ -192,7 +192,7 @@ export class TimeLine { } } else if (element.type == "lipsync") { - if(this.scrubberPosition+1 >= element.ending_offset){ + if(this.scrubberPosition+1 >= element.length){ this.lipSyncEngine.clips[element.object_uuid].stop(); } else if (object) { From 49d81e7955bf38fa97279d26643945e4119331e3 Mon Sep 17 00:00:00 2001 From: Michael Chung <6252325+ArEnSc@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:23:40 -0400 Subject: [PATCH 10/10] Fixed timeline styling --- app/pages/PageEnigma/js/editor.ts | 13 +-- app/pages/PageEnigma/js/timeline.ts | 135 ++++++++++++---------------- 2 files changed, 65 insertions(+), 83 deletions(-) diff --git a/app/pages/PageEnigma/js/editor.ts b/app/pages/PageEnigma/js/editor.ts index e5587d0422..f930d7e522 100644 --- a/app/pages/PageEnigma/js/editor.ts +++ b/app/pages/PageEnigma/js/editor.ts @@ -267,7 +267,6 @@ class Editor { this._test_demo(); - this.renderer.domElement.addEventListener("mousedown", this.onMouseDown.bind(this), false); this.renderer.domElement.addEventListener("mouseup", this.onMouseUp.bind(this), false); this.renderer.domElement.addEventListener("onContextMenu", this.onContextMenu.bind(this), false); @@ -321,6 +320,7 @@ class Editor { const result = await this.api_manager.getMediaBatch(["m_8fmp9hrvsqcryzka1fra597kg42s50","m_z4jzbst3xfh64h0qn4bqh4afenfps9"]); console.log(result); } + public async loadScene(scene_media_token: string) { this.dispatchAppUiState({ type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER, @@ -383,6 +383,7 @@ class Editor { `saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`, ); + // TODO turn scene information into and object ... const result = await this.api_manager.saveSceneState( this.activeScene.scene, name, @@ -390,7 +391,7 @@ class Editor { this.current_scene_media_token, new TimelineDataState(), ); - + if (result.data == null) { return; } @@ -755,7 +756,7 @@ class Editor { if (this.cameraViewControls && this.camera_person_mode) { this.cameraViewControls.update(5 * delta_time); if (this.cam_obj) { - if (this.timeline.isPlaying == false) { + if (this.timeline.is_playing == false) { this.cam_obj.position.copy(this.camera.position); this.cam_obj.rotation.copy(this.camera.rotation); } else { @@ -890,7 +891,7 @@ class Editor { let itteration = 0; - for (const clip of this.timeline.timelineItems) { + for (const clip of this.timeline.timeline_items) { if (clip.type == "lipsync" || clip.type == "audio") { await this.convertAudioClip(itteration, ffmpeg, clip); itteration += 1; @@ -932,7 +933,7 @@ class Editor { } startPlayback() { - this.timeline.isPlaying = true; + this.timeline.is_playing = true; this.timeline.scrubberPosition = 0; if (!this.camera_person_mode) { this.switchCameraView(); @@ -979,7 +980,7 @@ class Editor { let imgData = this.renderer.domElement.toDataURL(); this.frame_buffer.push(imgData); this.render_timer += this.clock.getDelta(); - if (this.timeline.isPlaying == false) { + if (this.timeline.is_playing == false) { this.stopPlayback(); this.playback_location = 0; this.rendering = false; diff --git a/app/pages/PageEnigma/js/timeline.ts b/app/pages/PageEnigma/js/timeline.ts index 143d4e25a4..6873675c84 100644 --- a/app/pages/PageEnigma/js/timeline.ts +++ b/app/pages/PageEnigma/js/timeline.ts @@ -6,88 +6,69 @@ import AudioEngine from "./audio_engine"; import TransformEngine from "./transform_engine"; import LipSyncEngine from "./lip_sync_engine"; import AnimationEngine from "./animation_engine"; -import { AudioClip } from "../models/track"; -import { AnimationClip } from "three"; -import { LipSyncClip } from "../datastructures/clips/lipsync_clip"; -import { TransformClip } from "../datastructures/clips/transform_clip"; // Every object uuid / entity has a track. export class TimelineCurrentState { - isEditable: boolean - selectedObjectID: number + is_editable: boolean + selected_object_ID: number constructor() { - this.isEditable = true // can add clips to it - this.selectedObjectID = 0 + this.is_editable = true // can add clips to it + this.selected_object_ID = 0 } } export class TimelineDataState { - timelineItems: ClipUI[] - scrubberPosition: number - - // Data to serialize with the scene, used to load into each engine. - audioClips:AudioClip[] - animationClips:AnimationClip[] - lipSyncClips:LipSyncClip[] - transformClips:TransformClip[] - - constructor(timelineItems: ClipUI[] = [], - audioClips: AudioClip[] = [], - animationClips: AnimationClip[] = [], - transformClips: TransformClip[] = [], - lipSyncClips: LipSyncClip[] = [], - scrubberPosition:number = 0) { - this.timelineItems = timelineItems - this.scrubberPosition = scrubberPosition - this.audioClips = audioClips - this.animationClips = animationClips - this.lipSyncClips = lipSyncClips - this.transformClips = transformClips + timeline_items: ClipUI[] + scrubber_frame_position: number + constructor(timeline_items: ClipUI[] = [], + scrubber_frame_position: number = 0) { + this.timeline_items = timeline_items + this.scrubber_frame_position = scrubber_frame_position } } export class TimeLine { - timelineItems: ClipUI[] + timeline_items: ClipUI[] - timeLineLimit: number - scrubberPosition: number - isPlaying: boolean + timeline_limit: number + scrubber_frame_position: number + is_playing: boolean // plays audio - audioEngine: AudioEngine + audio_engine: AudioEngine // key framing - transformEngine: TransformEngine + transform_engine: TransformEngine // animation engine - animationEngine: AnimationEngine + animation_engine: AnimationEngine // lip sync engine - lipSyncEngine: LipSyncEngine + lipSync_engine: LipSyncEngine scene: Scene // ensure that the elements are loaded first. - constructor(audioEngine: AudioEngine, - transformEngine: TransformEngine, - lipsyncEngine: LipSyncEngine, - animationEngine: AnimationEngine, + constructor(audio_engine: AudioEngine, + transform_engine: TransformEngine, + lipsync_engine: LipSyncEngine, + animation_engine: AnimationEngine, scene: Scene) { - this.timelineItems = [] - this.timeLineLimit = 60 * 5 // 5 seconds + this.timeline_items = [] + this.timeline_limit = 60 * 5 // 5 seconds - this.isPlaying = false - this.scrubberPosition = 0 // in frames into the tl + this.is_playing = false + this.scrubber_frame_position = 0 // in frames into the tl // this will be used to play the audio clips - this.audioEngine = audioEngine - this.transformEngine = transformEngine - this.lipSyncEngine = lipsyncEngine - this.animationEngine = animationEngine + this.audio_engine = audio_engine + this.transform_engine = transform_engine + this.lipSync_engine = lipsync_engine + this.animation_engine = animation_engine this.scene = scene; } public async addPlayableClip(clip: ClipUI): Promise { - this.timelineItems.push(clip) + this.timeline_items.push(clip) } // when given a media id item it will create the clip. @@ -130,28 +111,28 @@ export class TimeLine { } // public streaming events into the timeline from public async setScrubberPosition(offset: number) { - this.scrubberPosition = offset // in ms + this.scrubber_frame_position = offset // in ms } // should play from the clip that is closest to the to scrubber public async play(): Promise { console.log(`Starting Timeline`) - this.isPlaying = true + this.is_playing = true } - private async reset_scene() { - for (const element of this.timelineItems) { + private async resetScene() { + for (const element of this.timeline_items) { if (element.type == "transform") { let object = this.scene.get_object_by_uuid(element.object_uuid); - if (object && this.transformEngine.clips[element.object_uuid]) { this.transformEngine.clips[element.object_uuid].reset(object); } + if (object && this.transform_engine.clips[element.object_uuid]) { this.transform_engine.clips[element.object_uuid].reset(object); } } else if (element.type == "audio") { - this.audioEngine.loadClip(element.media_id); + this.audio_engine.loadClip(element.media_id); } else if (element.type == "animation") { } else if (element.type == "lipsync") { - this.lipSyncEngine.clips[element.object_uuid].reset(); + this.lipSync_engine.clips[element.object_uuid].reset(); } else { this.stop() @@ -162,48 +143,48 @@ export class TimeLine { // called by the editor update loop on each frame public async update(deltatime:number) { - if (this.isPlaying == false) return; // start and stop + if (this.is_playing == false) return; // start and stop - if (this.scrubberPosition <= 0) { - await this.reset_scene(); + if (this.scrubber_frame_position <= 0) { + await this.resetScene(); } - this.scrubberPosition += 1; + this.scrubber_frame_position += 1; //2. allow stopping. //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline. - for (const element of this.timelineItems) { - if (element.offset <= this.scrubberPosition && this.scrubberPosition <= element.length) { + for (const element of this.timeline_items) { + if (element.offset <= this.scrubber_frame_position && this.scrubber_frame_position <= element.length) { // run async // element.play() // remove the element from the list let object = this.scene.get_object_by_uuid(element.object_uuid) if (element.type == "transform") { - if (object && this.transformEngine.clips[element.object_uuid]) { - this.transformEngine.clips[element.object_uuid].length = (element.length - element.offset); - this.transformEngine.clips[element.object_uuid].step(object, element.offset, this.scrubberPosition); + if (object && this.transform_engine.clips[element.object_uuid]) { + this.transform_engine.clips[element.object_uuid].length = (element.length - element.offset); + this.transform_engine.clips[element.object_uuid].step(object, element.offset, this.scrubber_frame_position); } } else if (element.type == "audio") { - if(this.scrubberPosition+1 >= element.length){ - this.audioEngine.stopClip(element.media_id); + if(this.scrubber_frame_position+1 >= element.length){ + this.audio_engine.stopClip(element.media_id); } else { - this.audioEngine.playClip(element.media_id); + this.audio_engine.playClip(element.media_id); } } else if (element.type == "lipsync") { - if(this.scrubberPosition+1 >= element.length){ - this.lipSyncEngine.clips[element.object_uuid].stop(); + if(this.scrubber_frame_position+1 >= element.length){ + this.lipSync_engine.clips[element.object_uuid].stop(); } else if (object) { - await this.lipSyncEngine.clips[element.object_uuid].play(object); - this.lipSyncEngine.clips[element.object_uuid].step(); + await this.lipSync_engine.clips[element.object_uuid].play(object); + this.lipSync_engine.clips[element.object_uuid].step(); } } else if (element.type == "animation") { if (object) { - await this.animationEngine.clips[object.uuid].play(object); - this.animationEngine.clips[object.uuid].step(deltatime); + await this.animation_engine.clips[object.uuid].play(object); + this.animation_engine.clips[object.uuid].step(deltatime); } } else { @@ -214,14 +195,14 @@ export class TimeLine { } // find the offset of the longest clip and play until that clip is done - if (this.scrubberPosition >= this.timeLineLimit) { // stops at where clips should // cannot throw clip + if (this.scrubber_frame_position >= this.timeline_limit) { // stops at where clips should // cannot throw clip this.stop() } } } private async stop(): Promise { - this.isPlaying = false + this.is_playing = false console.log(`Stopping Timeline`) } }