Merge pull request #43 from storytold/feature/resize-timeline

Feature/resize timeline

app/pages/PageEnigma/PageEnigmaComponent.tsx

@@ -1,4 +1,4 @@
-import { useReducer, useEffect } from "react";
+import { useReducer } from "react";
 
 import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
 
@@ -14,18 +14,33 @@ import { SidePanelTabs } from "./comps/SidePanelTabs";
 
 import { EngineProvider } from "./contexts/EngineProvider";
 import { AppUIProvider } from "./contexts/AppUiContext";
-import { appUiReducer, appUiInitialStateValues, APPUI_ACTION_TYPES, APPUI_VIEW_MODES } from "./reducers";
+import {
+  appUiReducer,
+  appUiInitialStateValues,
+  APPUI_VIEW_MODES,
+} from "./reducers";
 import { ViewSideBySide } from "./comps/ViewSideBySide";
-import { timelineHeight } from "~/pages/PageEnigma/store";
+import {
+  canDrop,
+  clipLength,
+  currPosition,
+  dragId,
+  overTimeline,
+  scale,
+  timelineHeight,
+} from "~/pages/PageEnigma/store";
 import { useSignals } from "@preact/signals-react/runtime";
 
 export const PageEnigmaComponent = () => {
-
-  const [appUiState, dispatchAppUiState] = useReducer(appUiReducer, appUiInitialStateValues);
   useSignals();
+  const { currX, currY } = currPosition.value;
+
+  const [appUiState, dispatchAppUiState] = useReducer(
+    appUiReducer,
+    appUiInitialStateValues,
+  );
 
   const lowerHeight = timelineHeight.value;
-  console.log("main", timelineHeight.value);
   return (
     <div>
       <TopBarHelmet>
@@ -42,14 +57,14 @@ export const PageEnigmaComponent = () => {
               style={{ height: `calc(100% - ${lowerHeight}px)` }}
             >
               <div className="relative w-full overflow-hidden bg-transparent">
-
-                <div className={(appUiState.viewMode === APPUI_VIEW_MODES.SIDE_BY_SIDE) ? 'invisible' : ''}>
-                  <canvas
-                    id="video-scene"
-                    width="1280px"
-                    height="720px"
-                  />
-
+                <div
+                  className={
+                    appUiState.viewMode === APPUI_VIEW_MODES.SIDE_BY_SIDE
+                      ? "invisible"
+                      : ""
+                  }
+                >
+                  <canvas id="video-scene" width="1280px" height="720px" />
 
                   {/* Top controls */}
                   <div className="absolute left-0 top-0 w-full">
@@ -65,12 +80,11 @@ export const PageEnigmaComponent = () => {
                     <ControlsVideo />
                   </div>
                 </div>
-                {
-                  appUiState.viewMode === APPUI_VIEW_MODES.SIDE_BY_SIDE &&
+                {appUiState.viewMode === APPUI_VIEW_MODES.SIDE_BY_SIDE && (
                   <ViewSideBySide />
-                }
+                )}
                 <LoadingDots
-                  className="absolute top-0 left-0"
+                  className="absolute left-0 top-0"
                   isShowing={appUiState.showEditorLoader.isShowing}
                   type="bricks"
                   message={appUiState.showEditorLoader.message}
@@ -97,6 +111,22 @@ export const PageEnigmaComponent = () => {
           </div>
         </EngineProvider>
       </AppUIProvider>
+      <div
+        id={`ani-dnd-${dragId.value}`}
+        className={[
+          "absolute p-2",
+          "rounded-lg",
+          !canDrop.value ? "bg-brand-primary" : "bg-brand-secondary-700",
+          dragId.value ? "block" : "hidden",
+        ].join(" ")}
+        style={{
+          top: overTimeline.value ? currY - 16 : currY - 32,
+          left: currX + 1,
+          zIndex: 10000,
+          width: overTimeline.value ? clipLength.value * 4 * scale.value : 64,
+          height: overTimeline.value ? 32 : 64,
+        }}
+      />
     </div>
   );
-};
+};

app/pages/PageEnigma/Queue/Queue.ts

@@ -1,14 +1,21 @@
 import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
 import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
+import { QueueClip } from "~/pages/PageEnigma/models/track";
 
 class Queue {
   private _queue: Record<
     string,
-    { action: fromEngineActions | toEngineActions; data: any }[]
+    {
+      action: fromEngineActions | toEngineActions;
+      data: QueueClip | { currentTime: number };
+    }[]
   > = {};
   private _subscribers: Record<
     string,
-    (entry: { action: fromEngineActions | toEngineActions; data: any }) => void
+    (entry: {
+      action: fromEngineActions | toEngineActions;
+      data: QueueClip | { currentTime: number };
+    }) => void
   > = {};
 
   public publish({
@@ -18,12 +25,13 @@ class Queue {
   }: {
     queueName: string;
     action: fromEngineActions | toEngineActions;
-    data: any;
+    data: QueueClip | { currentTime: number };
   }) {
     if (!this._queue[queueName]) {
       this._queue[queueName] = [];
     }
     this._queue[queueName].push({ action, data });
+    console.log("Queued", queueName, action, data);
 
     if (this._subscribers[queueName]) {
       this._subscribers[queueName](this._queue[queueName].splice(0, 1)[0]);
@@ -34,7 +42,7 @@ class Queue {
     queueName: string,
     onMessage: (entry: {
       action: fromEngineActions | toEngineActions;
-      data: any;
+      data: QueueClip | { currentTime: number };
     }) => void,
   ) {
     this._subscribers[queueName] = onMessage;

app/pages/PageEnigma/Queue/fromEngineActions.ts

@@ -1,4 +1,4 @@
 export enum fromEngineActions {
-  FRAME_TICK, // data={currentTime: number}
-  OBJECT_ADDED, // data={id: string, name: string, ?????}
+  UPDATE_TIME = "update_time",
+  // data={currentTime: number}
 }

app/pages/PageEnigma/Queue/toEngineActions.ts

@@ -1,4 +1,18 @@
 export enum toEngineActions {
-  ADD_ANIMATION, // data={id: string, offset: number, length: number}
-  SCRUBBER_TIME, // data={currentTime: number}
+  ADD_CLIP = "add_clip",
+  // data=QueueClip
+  DELETE_CLIP = "delete_clip",
+  // data=QueueClip
+  MUTE = "mute",
+  // data={version, type, group, object_uuid} if lipsync
+  // data={version, type, group} if global_audio
+  PLAY_CLIP = "play_clip",
+  // data=QueueClip
+  UNMUTE = "unmute",
+  // data={version, type, group, object_uuid} if lipsync
+  // data={version, type, group} if global_audio
+  UPDATE_CLIP = "update_clip",
+  // data=QueueClip
+  UPDATE_TIME = "update_time",
+  // data={currentTime: number}
 }

app/pages/PageEnigma/comps/SidePanelTabs/ClipElement.tsx

@@ -1,33 +1,23 @@
-import { useCallback, useContext, useEffect, useState } from "react";
-import { BaseClip } from "~/pages/PageEnigma/models/track";
+import { useCallback, useContext, useEffect } from "react";
+import { MediaClip } from "~/pages/PageEnigma/models/track";
 import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
-import { canDrop, dragId, overTimeline, scale } from "~/pages/PageEnigma/store";
+import {
+  canDrop,
+  currPosition,
+  dragId,
+  initPosition,
+} from "~/pages/PageEnigma/store";
 import { useSignals } from "@preact/signals-react/runtime";
 
 interface Props {
-  clip: BaseClip;
+  clip: MediaClip;
   type: "animations" | "lipSync";
 }
 
 export const ClipElement = ({ clip, type }: Props) => {
   useSignals();
   const { startDrag, endDrag } = useContext(TrackContext);
-  const [initPosition, setInitPosition] = useState<{
-    initX: number;
-    initY: number;
-  }>({
-    initX: 0,
-    initY: 0,
-  });
-  const [currPosition, setCurrPosition] = useState<{
-    x: number;
-    y: number;
-  }>({
-    x: 0,
-    y: 0,
-  });
-  const { x, y } = currPosition;
-  const { initX, initY } = initPosition;
+  const { initX, initY } = initPosition.value;
 
   useEffect(() => {
     const onPointerUp = () => {
@@ -40,9 +30,9 @@ export const ClipElement = ({ clip, type }: Props) => {
       if (dragId) {
         event.stopPropagation();
         event.preventDefault();
-        const deltaX = event.clientX - initX;
-        const deltaY = event.clientY - initY;
-        setCurrPosition({ x: deltaX, y: deltaY });
+        const deltaX = event.pageX - initX;
+        const deltaY = event.pageY - initY;
+        currPosition.value = { currX: initX + deltaX, currY: initY + deltaY };
         return;
       }
     };
@@ -59,53 +49,31 @@ export const ClipElement = ({ clip, type }: Props) => {
   const onPointerDown = useCallback(
     (event: React.PointerEvent<HTMLDivElement>) => {
       if (event.button === 0) {
-        const box = document.getElementById(`ani-clip-${clip.id}`);
-        const position = box!.getBoundingClientRect();
-
-        startDrag(type, clip.id, clip.length);
-        setCurrPosition({
-          x: position.x,
-          y: position.y + position.height + 1,
-        });
-        setInitPosition({
-          initX: position.x,
-          initY: position.y + position.height + 1,
-        });
+        startDrag(type, clip.media_id, clip.length);
+        currPosition.value = {
+          currX: event.pageX,
+          currY: event.pageY,
+        };
+        initPosition.value = {
+          initX: event.pageX,
+          initY: event.pageY,
+        };
         canDrop.value = false;
       }
     },
-    [type, clip.id, startDrag],
+    [type, clip.media_id, startDrag, clip.length],
   );
 
   return (
-    <div key={clip.id} className="relative h-16 w-16">
+    <div className="relative h-16 w-16">
       <div
-        id={`ani-clip-${clip.id}`}
-        className="absolute block h-16 w-16 bg-brand-secondary-700"
+        id={`ani-clip-${clip.media_id}`}
+        className="absolute block h-16 w-16 rounded-lg bg-brand-secondary-700 p-2"
         style={{ top: 0, left: 0 }}
         onPointerDown={onPointerDown}
       >
         {clip.name}
       </div>
-      <div
-        id={`ani-dnd-${clip.id}`}
-        className={[
-          "absolute p-2",
-          "rounded-lg",
-          !canDrop.value ? "bg-brand-primary" : "bg-brand-secondary-700",
-          dragId.value ? "block" : "hidden",
-        ].join(" ")}
-        style={{
-          top: overTimeline.value ? y + 24 : y,
-          left: overTimeline.value ? x : x,
-          zIndex: 5000,
-          width: overTimeline.value ? clip.length * 4 * scale.value : 64,
-          height: overTimeline.value ? 32 : 64,
-        }}
-        onPointerDown={onPointerDown}
-      >
-        {clip.name}
-      </div>
     </div>
   );
 };

app/pages/PageEnigma/comps/SidePanelTabs/ClipElements.tsx

@@ -1,14 +1,24 @@
-import { useContext } from "react";
+import { useContext, useEffect, useState } from "react";
 import { ClipElement } from "./ClipElement";
 import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
+import { useSignals } from "@preact/signals-react/runtime";
+import { timelineHeight } from "~/pages/PageEnigma/store";
 
 export const ClipElements = () => {
   const { animationClips } = useContext(TrackContext);
+  useSignals();
+  const [height, setHeight] = useState(0);
+
+  useEffect(() => {
+    setHeight(window.outerHeight - timelineHeight.value);
+  }, []);
 
   return (
-    <div className="flex flex-wrap">
+    <div className="flex flex-wrap gap-3 overflow-y-auto" style={{ height }}>
       {animationClips.map((clip) => {
-        return <ClipElement key={clip.id} clip={clip} type="animations" />;
+        return (
+          <ClipElement key={clip.media_id} clip={clip} type="animations" />
+        );
       })}
     </div>
   );

app/pages/PageEnigma/comps/Timeline/Timeline.tsx

@@ -21,7 +21,7 @@ import { EngineContext } from "../../contexts/EngineContext";
 
 export const Timeline = () => {
   const editorEngine = useContext(EngineContext);
-    
+
   const {
     characters,
     objects,
@@ -148,7 +148,7 @@ export const Timeline = () => {
             style={{
               left: 0,
               top: 8,
-              width: 2,
+              width: 4,
               height: fullHeight,
             }}
           />

app/pages/PageEnigma/comps/Timeline/TrackClips.tsx

@@ -1,7 +1,7 @@
 import { TrackClip } from "~/pages/PageEnigma/comps/Timeline/TrackClip";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faVolume, faVolumeSlash } from "@fortawesome/pro-solid-svg-icons";
-import { BaseClip } from "~/pages/PageEnigma/models/track";
+import { Clip } from "~/pages/PageEnigma/models/track";
 import { PointerEvent } from "react";
 import {
   canDrop,
@@ -15,7 +15,7 @@ import {
 
 interface Props {
   id: string;
-  clips: BaseClip[];
+  clips: Clip[];
   title: string;
   style: "character" | "audio" | "camera" | "objects";
   type?: "animations" | "positions" | "lipSync";
@@ -94,7 +94,7 @@ export const TrackClips = ({
       >
         {clips.map((clip, index) => (
           <TrackClip
-            key={clip.id}
+            key={clip.clip_uuid}
             min={
               index > 0 ? clips[index - 1].offset + clips[index - 1].length : 0
             }

app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsAnimation.ts

@@ -20,7 +20,7 @@ export const useMouseEventsAnimation = () => {
         setTime(-1);
         Queue.publish({
           queueName: QueueNames.TO_ENGINE,
-          action: toEngineActions.SCRUBBER_TIME,
+          action: toEngineActions.UPDATE_TIME,
           data: { currentTime: Math.round(time) },
         });
       }

app/pages/PageEnigma/comps/Timeline/utils/useMouseEventsClip.ts

@@ -1,10 +1,10 @@
 import React, { Dispatch, useCallback, useEffect, useRef } from "react";
-import { BaseClip } from "~/pages/PageEnigma/models/track";
+import { Clip } from "~/pages/PageEnigma/models/track";
 import { canDrop, scale } from "~/pages/PageEnigma/store";
 import { useSignals } from "@preact/signals-react/runtime";
 
 export const useMouseEventsClip = (
-  clip: BaseClip,
+  clip: Clip,
   max: number,
   min: number,
   updateClip: (args: { id: string; offset: number; length: number }) => void,
@@ -21,21 +21,26 @@ export const useMouseEventsClip = (
   const onPointerUp = useCallback(() => {
     if (isActive.current) {
       updateClip({
-        id: clip.id,
+        id: clip.clip_uuid,
         offset: Math.round(currOffset.current),
         length: Math.round(currLength.current),
       });
       isActive.current = "";
       canDrop.value = false;
     }
-  }, [updateClip, clip.id]);
+  }, [updateClip, clip.clip_uuid]);
 
   const onMouseMove = useCallback(
     (event: MouseEvent) => {
       const delta = (event.clientX - clientX.current) / 4 / scale.value;
       const deltaOffset = delta + initOffset.current;
       if (isActive.current === "drag") {
-        if (deltaOffset < min || deltaOffset + currLength.current > max) {
+        if (deltaOffset < min) {
+          currOffset.current = min;
+          return;
+        }
+        if (deltaOffset + currLength.current > max) {
+          currOffset.current = max - currLength.current;
           return;
         }
         currOffset.current = deltaOffset;

app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts

@@ -1,28 +1,27 @@
 import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
 import { useSignals } from "@preact/signals-react/runtime";
-import { useCallback, useContext, useEffect } from "react";
+import { useCallback, useEffect } from "react";
 import { currentTime } from "~/pages/PageEnigma/store";
-import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
 import Queue from "~/pages/PageEnigma/Queue/Queue";
 import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
 import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
+import { QueueClip } from "~/pages/PageEnigma/models/track";
+
+interface UpdateTime {
+  currentTime: number;
+}
 
 interface Arguments {
   action: fromEngineActions | toEngineActions;
-  data: any;
+  data: QueueClip | UpdateTime;
 }
 
 export function useQueueHandler() {
   useSignals();
-  const { addObject } = useContext(TrackContext);
-
   const handleActions = useCallback(({ action, data }: Arguments) => {
     switch (action) {
-      case fromEngineActions.FRAME_TICK:
-        currentTime.value += 1;
-        break;
-      case fromEngineActions.OBJECT_ADDED:
-        addObject(data);
+      case fromEngineActions.UPDATE_TIME:
+        currentTime.value = (data as UpdateTime).currentTime;
         break;
       default:
         throw new Error(`Unknown action ${action}`);

app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx

@@ -1,13 +1,11 @@
 import { createContext } from "react";
 import {
-  AnimationClip,
-  AudioClip,
   AudioGroup,
-  BaseClip,
   CameraGroup,
   CharacterGroup,
   ObjectGroup,
   ObjectTrack,
+  MediaClip,
 } from "~/pages/PageEnigma/models/track";
 
 export const TrackContext = createContext<{
@@ -23,13 +21,13 @@ export const TrackContext = createContext<{
   addCharacterAnimation: (options: {
     clipId: string;
     characterId: string;
-    animationClips: BaseClip[];
+    animationClips: MediaClip[];
     offset: number;
   }) => void;
   addCharacterAudio: (options: {
     clipId: string;
     characterId: string;
-    audioClips: BaseClip[];
+    audioClips: MediaClip[];
     offset: number;
   }) => void;
   selectCharacterClip: (clipId: string) => void;
@@ -55,7 +53,7 @@ export const TrackContext = createContext<{
   toggleAudioMute: () => void;
   addGlobalAudio: (
     clipId: string,
-    animationClips: BaseClip[],
+    animationClips: MediaClip[],
     offset: number,
   ) => void;
   selectAudioClip: (clipId: string) => void;
@@ -71,8 +69,8 @@ export const TrackContext = createContext<{
   selectClip: (clipId: string) => void;
 
   // sidebar clips
-  animationClips: AnimationClip[];
-  audioClips: AudioClip[];
+  animationClips: MediaClip[];
+  audioClips: MediaClip[];
 
   // drag and drop
   startDrag: (

app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx

@@ -4,7 +4,7 @@ import useUpdateCharacters from "~/pages/PageEnigma/contexts/TrackContext/utils/
 import useUpdateCamera from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera";
 import useUpdateAudio from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio";
 import useUpdateObject from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateObject";
-import { AnimationClip, AudioClip } from "~/pages/PageEnigma/models/track";
+import { ClipType, MediaClip } from "~/pages/PageEnigma/models/track";
 import useUpdateDragDrop from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop";
 import {
   canDrop,
@@ -13,6 +13,7 @@ import {
   dropId,
   dropOffset,
 } from "~/pages/PageEnigma/store";
+import * as uuid from "uuid";
 
 interface Props {
   children: ReactNode;
@@ -31,8 +32,8 @@ export const TrackProvider = ({ children }: Props) => {
     setSelectedClip(clipId);
   }, []);
 
-  const [animationClips, setAnimationClips] = useState<AnimationClip[]>([]);
-  const [audioClips, setAudioClips] = useState<AudioClip[]>([]);
+  const [animationClips, setAnimationClips] = useState<MediaClip[]>([]);
+  const [audioClips, setAudioClips] = useState<MediaClip[]>([]);
 
   const [scale, setScale] = useState(1);
   const [length, setLength] = useState(1);
@@ -74,26 +75,30 @@ export const TrackProvider = ({ children }: Props) => {
   useEffect(() => {
     setAnimationClips([
       {
-        id: "ani-id1",
-        offset: 0,
+        version: 1,
+        media_id: uuid.v4(),
+        type: ClipType.ANIMATION,
         length: 25,
         name: "Sit",
       },
       {
-        id: "ani-id2",
-        offset: 0,
+        version: 1,
+        media_id: uuid.v4(),
+        type: ClipType.ANIMATION,
         length: 25,
         name: "Idle",
       },
       {
-        id: "ani-id3",
-        offset: 0,
+        version: 1,
+        media_id: uuid.v4(),
+        type: ClipType.ANIMATION,
         length: 25,
         name: "Stand",
       },
       {
-        id: "ani-id4",
-        offset: 0,
+        version: 1,
+        media_id: uuid.v4(),
+        type: ClipType.ANIMATION,
         length: 25,
         name: "Walk",
       },

app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateAudio.ts

@@ -1,24 +1,21 @@
 import { useCallback, useState } from "react";
-import { AudioGroup, BaseClip } from "~/pages/PageEnigma/models/track";
+import {
+  AudioGroup,
+  Clip,
+  ClipGroup,
+  ClipType,
+  MediaClip,
+} from "~/pages/PageEnigma/models/track";
+import Queue from "~/pages/PageEnigma/Queue/Queue";
+import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
+import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
+import * as uuid from "uuid";
 
 export default function useUpdateAudio() {
   const [audio, setAudio] = useState<AudioGroup>({
     id: "AU1",
     muted: false,
-    clips: [
-      {
-        id: "AU1-1",
-        length: 200,
-        offset: 0,
-        name: "audio 1",
-      },
-      {
-        id: "AU1-2",
-        length: 180,
-        offset: 300,
-        name: "audio 2",
-      },
-    ],
+    clips: [],
   });
 
   const updateAudio = useCallback(
@@ -33,44 +30,56 @@ export default function useUpdateAudio() {
     }) => {
       setAudio((oldAudio) => {
         const newClips = [...oldAudio.clips];
-        const clipIndex = newClips.findIndex((row) => row.id === id);
+        const clipIndex = newClips.findIndex((row) => row.clip_uuid === id);
         if (clipIndex === -1) {
           return { ...oldAudio };
         }
         const clip = newClips[clipIndex];
         clip.offset = offset;
         clip.length = length;
+
+        Queue.publish({
+          queueName: QueueNames.TO_ENGINE,
+          action: toEngineActions.UPDATE_CLIP,
+          data: clip,
+        });
+
         return {
           ...oldAudio,
           clips: newClips,
         };
       });
-      console.log("message", {
-        action: "UpdateGlobalAudio",
-        id,
-        data: { offset, length },
-      });
     },
     [],
   );
 
   const addGlobalAudio = useCallback(
-    (dragId: string, audioClips: BaseClip[], offset: number) => {
-      const clip = audioClips.find((row) => row.id === dragId);
+    (dragId: string, audioClips: MediaClip[], offset: number) => {
+      const clip = audioClips.find((row) => row.media_id === dragId);
       if (!clip) {
         return;
       }
 
+      const clip_uuid = uuid.v4();
+      const newClip = {
+        ...clip,
+        type: ClipType.AUDIO,
+        group: ClipGroup.GLOBAL_AUDIO,
+        offset,
+        clip_uuid,
+      } as Clip;
+
       setAudio((oldAudio) => {
         return {
           ...oldAudio,
-          clips: [...oldAudio.clips, { ...clip, offset }],
+          clips: [...oldAudio.clips, newClip],
         };
       });
-      console.log("message", {
-        action: "UpdateGlobalAudio",
-        id: dragId,
-        data: { offset },
+
+      Queue.publish({
+        queueName: QueueNames.TO_ENGINE,
+        action: toEngineActions.ADD_CLIP,
+        data: newClip,
       });
     },
     [],
@@ -78,15 +87,21 @@ export default function useUpdateAudio() {
 
   const toggleAudioMute = useCallback(() => {
     setAudio((oldAudio) => {
+      Queue.publish({
+        queueName: QueueNames.TO_ENGINE,
+        action: oldAudio?.muted ? toEngineActions.UNMUTE : toEngineActions.MUTE,
+        data: {
+          version: 1,
+          type: ClipType.AUDIO,
+          group: ClipGroup.GLOBAL_AUDIO,
+        },
+      });
+
       return {
         ...oldAudio,
         muted: !oldAudio.muted,
       };
     });
-    console.log("message", {
-      action: "ToggleGlobalAudioMute",
-      id: "",
-    });
   }, []);
 
   const selectAudioClip = useCallback((clipId: string) => {
@@ -97,7 +112,8 @@ export default function useUpdateAudio() {
           ...oldAudio.clips.map((clip) => {
             return {
               ...clip,
-              selected: clip.id === clipId ? !clip.selected : clip.selected,
+              selected:
+                clip.clip_uuid === clipId ? !clip.selected : clip.selected,
             };
           }),
         ],
@@ -109,13 +125,21 @@ export default function useUpdateAudio() {
     setAudio((oldAudio) => {
       return {
         ...oldAudio,
-        clips: [...oldAudio.clips.filter((clip) => clip.id !== clipId)],
+        clips: [
+          ...oldAudio.clips.filter((clip) => {
+            if (clip.clip_uuid === clipId) {
+              Queue.publish({
+                queueName: QueueNames.TO_ENGINE,
+                action: toEngineActions.DELETE_CLIP,
+                data: clip!,
+              });
+              return false;
+            }
+            return true;
+          }),
+        ],
       };
     });
-    console.log("message", {
-      action: "UpdateGlobalAudio",
-      id: clipId,
-    });
   }, []);
 
   return {

app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCamera.ts

@@ -1,23 +1,13 @@
 import { useCallback, useState } from "react";
 import { CameraGroup } from "~/pages/PageEnigma/models/track";
+import Queue from "~/pages/PageEnigma/Queue/Queue";
+import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
+import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
 
 export default function useUpdateCamera() {
   const [camera, setCamera] = useState<CameraGroup>({
     id: "CA1",
-    clips: [
-      {
-        id: "CA1-1",
-        length: 200,
-        offset: 0,
-        name: "cam 1",
-      },
-      {
-        id: "CA1-2",
-        length: 180,
-        offset: 300,
-        name: "cam 2",
-      },
-    ],
+    clips: [],
   });
 
   const updateCamera = useCallback(
@@ -32,13 +22,20 @@ export default function useUpdateCamera() {
     }) => {
       setCamera((oldCamera) => {
         const newClips = [...oldCamera.clips];
-        const clipIndex = newClips.findIndex((row) => row.id === id);
+        const clipIndex = newClips.findIndex((row) => row.clip_uuid === id);
         if (clipIndex === -1) {
           return { ...oldCamera };
         }
         const clip = newClips[clipIndex];
         clip.offset = offset;
         clip.length = length;
+
+        Queue.publish({
+          queueName: QueueNames.TO_ENGINE,
+          action: toEngineActions.UPDATE_CLIP,
+          data: clip,
+        });
+
         return {
           ...oldCamera,
           clips: newClips,
@@ -56,7 +53,8 @@ export default function useUpdateCamera() {
           ...oldCamera.clips.map((clip) => {
             return {
               ...clip,
-              selected: clip.id === clipId ? !clip.selected : clip.selected,
+              selected:
+                clip.clip_uuid === clipId ? !clip.selected : clip.selected,
             };
           }),
         ],
@@ -68,7 +66,19 @@ export default function useUpdateCamera() {
     setCamera((oldCamera) => {
       return {
         ...oldCamera,
-        clips: [...oldCamera.clips.filter((clip) => clip.id !== clipId)],
+        clips: [
+          ...oldCamera.clips.filter((clip) => {
+            if (clip.clip_uuid === clipId) {
+              Queue.publish({
+                queueName: QueueNames.TO_ENGINE,
+                action: toEngineActions.DELETE_CLIP,
+                data: clip!,
+              });
+              return false;
+            }
+            return true;
+          }),
+        ],
       };
     });
   }, []);

app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateCharacters.ts

@@ -1,5 +1,11 @@
 import { useCallback, useState } from "react";
-import { BaseClip, CharacterGroup } from "~/pages/PageEnigma/models/track";
+import {
+  Clip,
+  CharacterGroup,
+  ClipGroup,
+  ClipType,
+  MediaClip,
+} from "~/pages/PageEnigma/models/track";
 import * as uuid from "uuid";
 import Queue from "~/pages/PageEnigma/Queue/Queue";
 import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
@@ -12,16 +18,10 @@ export default function useUpdateCharacters() {
       muted: false,
       animationClips: [],
       positionClips: [],
-      lipSyncClips: [
-        {
-          id: "CH1-L1",
-          length: 400,
-          offset: 150,
-          name: "sync 1",
-        },
-      ],
+      lipSyncClips: [],
     },
   ]);
+
   const updateCharacters = useCallback(
     ({
       type,
@@ -39,7 +39,7 @@ export default function useUpdateCharacters() {
           return oldCharacters.map((character) => {
             const newAnimationClips = [...character.animationClips];
             const clipIndex = newAnimationClips.findIndex(
-              (row) => row.id === id,
+              (row) => row.clip_uuid === id,
             );
             if (clipIndex === -1) {
               return { ...character };
@@ -47,25 +47,28 @@ export default function useUpdateCharacters() {
             const clip = newAnimationClips[clipIndex];
             clip.offset = offset;
             clip.length = length;
+
+            Queue.publish({
+              queueName: QueueNames.TO_ENGINE,
+              action: toEngineActions.UPDATE_CLIP,
+              data: clip,
+            });
+
             return {
               ...character,
               animationClips: newAnimationClips,
             };
           });
         });
-        console.log("message", {
-          action: "UpdateAnimation",
-          id,
-          data: { offset, length },
-        });
         return;
       }
+
       if (type === "positions") {
         setCharacters((oldCharacters) => {
           return oldCharacters.map((character) => {
             const newPositionClips = [...character.positionClips];
             const clipIndex = newPositionClips.findIndex(
-              (row) => row.id === id,
+              (row) => row.clip_uuid === id,
             );
             if (clipIndex === -1) {
               return { ...character };
@@ -73,6 +76,13 @@ export default function useUpdateCharacters() {
             const clip = newPositionClips[clipIndex];
             clip.offset = offset;
             clip.length = length;
+
+            Queue.publish({
+              queueName: QueueNames.TO_ENGINE,
+              action: toEngineActions.UPDATE_CLIP,
+              data: clip,
+            });
+
             return {
               ...character,
               positionClips: newPositionClips,
@@ -85,24 +95,28 @@ export default function useUpdateCharacters() {
         setCharacters((oldCharacters) => {
           return oldCharacters.map((character) => {
             const newLipSyncClips = [...character.lipSyncClips];
-            const clipIndex = newLipSyncClips.findIndex((row) => row.id === id);
+            const clipIndex = newLipSyncClips.findIndex(
+              (row) => row.clip_uuid === id,
+            );
             if (clipIndex === -1) {
               return { ...character };
             }
             const clip = newLipSyncClips[clipIndex];
             clip.offset = offset;
             clip.length = length;
+
+            Queue.publish({
+              queueName: QueueNames.TO_ENGINE,
+              action: toEngineActions.UPDATE_CLIP,
+              data: clip,
+            });
+
             return {
               ...character,
               lipSyncClips: newLipSyncClips,
             };
           });
         });
-        console.log("message", {
-          action: "UpdateLipSync",
-          id,
-          data: { offset, length },
-        });
       }
     },
     [],
@@ -117,14 +131,23 @@ export default function useUpdateCharacters() {
     }: {
       clipId: string;
       characterId: string;
-      animationClips: BaseClip[];
+      animationClips: MediaClip[];
       offset: number;
     }) => {
-      const clip = animationClips.find((row) => row.id === clipId);
+      const clip = animationClips.find((row) => row.media_id === clipId);
       if (!clip) {
         return;
       }
 
+      const clip_uuid = uuid.v4();
+      const newClip = {
+        ...clip,
+        group: ClipGroup.CHARACTER,
+        offset,
+        clip_uuid,
+        object_uuid: characterId,
+      } as Clip;
+
       setCharacters((oldCharacters) => {
         return oldCharacters.map((character) => {
           if (character.id !== characterId) {
@@ -132,20 +155,17 @@ export default function useUpdateCharacters() {
           }
           return {
             ...character,
-            animationClips: [
-              ...character.animationClips,
-              { ...clip, id: uuid.v4(), offset },
-            ],
+            animationClips: [...character.animationClips, newClip].sort(
+              (clipA, clipB) => clipA.offset - clipB.offset,
+            ),
           };
         });
       });
+
       Queue.publish({
         queueName: QueueNames.TO_ENGINE,
-        action: toEngineActions.ADD_ANIMATION,
-        data: {
-          clipId,
-          offset,
-        },
+        action: toEngineActions.ADD_CLIP,
+        data: newClip,
       });
     },
     [],
@@ -160,14 +180,24 @@ export default function useUpdateCharacters() {
     }: {
       clipId: string;
       characterId: string;
-      audioClips: BaseClip[];
+      audioClips: MediaClip[];
       offset: number;
     }) => {
-      const clip = audioClips.find((row) => row.id === clipId);
+      const clip = audioClips.find((row) => row.media_id === clipId);
       if (!clip) {
         return;
       }
 
+      const clip_uuid = uuid.v4();
+      const newClip = {
+        ...clip,
+        type: ClipType.AUDIO,
+        group: ClipGroup.CHARACTER,
+        offset,
+        clip_uuid,
+        object_uuid: characterId,
+      } as Clip;
+
       setCharacters((oldCharacters) => {
         return oldCharacters.map((character) => {
           if (character.id !== characterId) {
@@ -175,14 +205,17 @@ export default function useUpdateCharacters() {
           }
           return {
             ...character,
-            lipSyncClips: [...character.lipSyncClips, { ...clip, offset }],
+            lipSyncClips: [...character.lipSyncClips, newClip].sort(
+              (clipA, clipB) => clipA.offset - clipB.offset,
+            ),
           };
         });
       });
-      console.log("message", {
-        action: "AddLipSync",
-        id: clipId,
-        data: { offset },
+
+      Queue.publish({
+        queueName: QueueNames.TO_ENGINE,
+        action: toEngineActions.ADD_CLIP,
+        data: newClip,
       });
     },
     [],
@@ -190,15 +223,28 @@ export default function useUpdateCharacters() {
 
   const toggleLipSyncMute = useCallback((characterId: string) => {
     setCharacters((oldCharacters) => {
-      return oldCharacters.map((character) => ({
-        ...character,
-        muted:
-          character.id === characterId ? !character.muted : character.muted,
-      }));
-    });
-    console.log("message", {
-      action: "ToggleLipSync",
-      id: characterId,
+      return oldCharacters.map((character) => {
+        if (character.id === characterId) {
+          Queue.publish({
+            queueName: QueueNames.TO_ENGINE,
+            action: character?.muted
+              ? toEngineActions.UNMUTE
+              : toEngineActions.MUTE,
+            data: {
+              version: 1,
+              type: ClipType.AUDIO,
+              group: ClipGroup.CHARACTER,
+              object_uuid: characterId,
+            },
+          });
+        }
+
+        return {
+          ...character,
+          muted:
+            character.id === characterId ? !character.muted : character.muted,
+        };
+      });
     });
   }, []);
 
@@ -209,15 +255,18 @@ export default function useUpdateCharacters() {
           ...character,
           animationClips: character.animationClips.map((clip) => ({
             ...clip,
-            selected: clip.id === clipId ? !clip.selected : clip.selected,
+            selected:
+              clip.clip_uuid === clipId ? !clip.selected : clip.selected,
           })),
           positionClips: character.positionClips.map((clip) => ({
             ...clip,
-            selected: clip.id === clipId ? !clip.selected : clip.selected,
+            selected:
+              clip.clip_uuid === clipId ? !clip.selected : clip.selected,
           })),
           lipSyncClips: character.lipSyncClips.map((clip) => ({
             ...clip,
-            selected: clip.id === clipId ? !clip.selected : clip.selected,
+            selected:
+              clip.clip_uuid === clipId ? !clip.selected : clip.selected,
           })),
         })),
       ];
@@ -229,22 +278,42 @@ export default function useUpdateCharacters() {
       return [
         ...oldCharacters.map((character) => ({
           ...character,
-          animationClips: character.animationClips.filter(
-            (clip) => clip.id !== clipId,
-          ),
-          positionClips: character.positionClips.filter(
-            (clip) => clip.id !== clipId,
-          ),
-          lipSyncClips: character.lipSyncClips.filter(
-            (clip) => clip.id !== clipId,
-          ),
+          animationClips: character.animationClips.filter((clip) => {
+            if (clip.clip_uuid === clipId) {
+              Queue.publish({
+                queueName: QueueNames.TO_ENGINE,
+                action: toEngineActions.DELETE_CLIP,
+                data: clip!,
+              });
+              return false;
+            }
+            return true;
+          }),
+          positionClips: character.positionClips.filter((clip) => {
+            if (clip.clip_uuid === clipId) {
+              Queue.publish({
+                queueName: QueueNames.TO_ENGINE,
+                action: toEngineActions.DELETE_CLIP,
+                data: clip!,
+              });
+              return false;
+            }
+            return true;
+          }),
+          lipSyncClips: character.lipSyncClips.filter((clip) => {
+            if (clip.clip_uuid === clipId) {
+              Queue.publish({
+                queueName: QueueNames.TO_ENGINE,
+                action: toEngineActions.DELETE_CLIP,
+                data: clip!,
+              });
+              return false;
+            }
+            return true;
+          }),
         })),
       ];
     });
-    console.log("message", {
-      action: "DeleteCharacterClip",
-      id: clipId,
-    });
   }, []);
 
   return {

app/pages/PageEnigma/models/track.ts

@@ -1,11 +1,50 @@
-export interface BaseClip {
-  id: string;
+export interface QueueClip {
+  version: number;
+  type: ClipType;
+  group: ClipGroup;
+  object_uuid?: string;
+  clip_uuid?: string;
+  media_id?: string;
+  name?: string;
+  offset?: number;
+  length?: number;
+  selected?: boolean;
+}
+
+export interface MediaClip {
+  version: number;
+  type: ClipType;
+  media_id: string;
+  name: string;
+  length: number;
+}
+
+export interface Clip {
+  version: number;
+  clip_uuid: string;
+  type: ClipType;
+  group: ClipGroup;
+  media_id: string;
+  object_uuid?: string;
   name: string;
   offset: number;
   length: number;
   selected?: boolean;
 }
 
+export enum ClipType {
+  TRANSFORM = "transform",
+  AUDIO = "audio",
+  ANIMATION = "animation",
+}
+
+export enum ClipGroup {
+  CHARACTER = "character",
+  CAMERA = "camera",
+  GLOBAL_AUDIO = "global_audio",
+  OBJECT = "object",
+}
+
 export interface BaseKeyFrame {
   id: string;
   name: string;
@@ -13,35 +52,25 @@ export interface BaseKeyFrame {
   selected?: boolean;
 }
 
-export interface AnimationClip extends BaseClip {}
-
-export interface PositionClip extends BaseClip {}
-
-export interface LipSyncClip extends BaseClip {}
-
 export interface CharacterGroup {
   id: string;
   muted: boolean;
-  animationClips: AnimationClip[];
-  positionClips: PositionClip[];
-  lipSyncClips: LipSyncClip[];
+  animationClips: Clip[];
+  positionClips: Clip[];
+  lipSyncClips: Clip[];
 }
 
 export interface CameraGroup {
   id: string;
-  clips: CameraClip[];
+  clips: Clip[];
 }
 
-export interface CameraClip extends BaseClip {}
-
 export interface AudioGroup {
   id: string;
   muted: boolean;
-  clips: AudioClip[];
+  clips: Clip[];
 }
 
-export interface AudioClip extends BaseClip {}
-
 export interface ObjectGroup {
   id: string;
   objects: ObjectTrack[];

app/pages/PageEnigma/store/dragAndDrop.ts

@@ -7,3 +7,11 @@ export const overTimeline = signal(false);
 export const dropId = signal("");
 export const dropOffset = signal(0);
 export const clipLength = signal(0);
+export const initPosition = signal<{ initX: number; initY: number }>({
+  initX: 0,
+  initY: 0,
+});
+export const currPosition = signal<{ currX: number; currY: number }>({
+  currX: 0,
+  currY: 0,
+});

app/root.tsx

@@ -107,4 +107,4 @@ function CompleteTakeoverLoadingScreen({isShowing}:{isShowing:boolean}){
       <LoadingDotsBricks />
     </Transition>
   );
-};
+};