Merge pull request #72 from storytold/feature/keyframes

Keyframe drag, keyframe display, audio dnd

app/pages/PageEnigma/comps/SidePanelTabs/AnimationElement.tsx

@@ -11,10 +11,10 @@ import { useSignals } from "@preact/signals-react/runtime";
 
 interface Props {
   clip: MediaClip;
-  type: "animations" | "lipSync";
+  type: "animations" | "audio";
 }
 
-export const ClipElement = ({ clip, type }: Props) => {
+export const AnimationElement = ({ clip, type }: Props) => {
   useSignals();
   const { startDrag, endDrag } = useContext(TrackContext);
   const { initX, initY } = initPosition.value;
@@ -33,7 +33,6 @@ export const ClipElement = ({ clip, type }: Props) => {
         const deltaX = event.pageX - initX;
         const deltaY = event.pageY - initY;
         currPosition.value = { currX: initX + deltaX, currY: initY + deltaY };
-        return;
       }
     };
 

app/pages/PageEnigma/comps/SidePanelTabs/AnimationElements.tsx

@@ -1,10 +1,10 @@
 import { useContext, useEffect, useState } from "react";
-import { ClipElement } from "./ClipElement";
+import { AnimationElement } from "./AnimationElement";
 import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
 import { useSignals } from "@preact/signals-react/runtime";
 import { timelineHeight } from "~/pages/PageEnigma/store";
 
-export const ClipElements = () => {
+export const AnimationElements = () => {
   const { animationClips } = useContext(TrackContext);
   useSignals();
   const [height, setHeight] = useState(0);
@@ -17,7 +17,7 @@ export const ClipElements = () => {
     <div className="flex flex-wrap gap-3 overflow-y-auto" style={{ height }}>
       {animationClips.map((clip) => {
         return (
-          <ClipElement key={clip.media_id} clip={clip} type="animations" />
+          <AnimationElement key={clip.media_id} clip={clip} type="animations" />
         );
       })}
     </div>

app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx

@@ -0,0 +1,78 @@
+import { useCallback, useContext, useEffect } from "react";
+import { MediaClip } from "~/pages/PageEnigma/models/track";
+import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
+import {
+  canDrop,
+  currPosition,
+  dragId,
+  initPosition,
+} from "~/pages/PageEnigma/store";
+import { useSignals } from "@preact/signals-react/runtime";
+
+interface Props {
+  clip: MediaClip;
+  type: "animations" | "audio";
+}
+
+export const AudioElement = ({ clip, type }: Props) => {
+  useSignals();
+  const { startDrag, endDrag } = useContext(TrackContext);
+  const { initX, initY } = initPosition.value;
+
+  useEffect(() => {
+    const onPointerUp = () => {
+      if (dragId.value) {
+        endDrag();
+      }
+    };
+
+    const onMouseMove = (event: MouseEvent) => {
+      if (dragId) {
+        event.stopPropagation();
+        event.preventDefault();
+        const deltaX = event.pageX - initX;
+        const deltaY = event.pageY - initY;
+        currPosition.value = { currX: initX + deltaX, currY: initY + deltaY };
+      }
+    };
+
+    window.addEventListener("pointerup", onPointerUp);
+    window.addEventListener("pointermove", onMouseMove);
+
+    return () => {
+      window.removeEventListener("pointerup", onPointerUp);
+      window.removeEventListener("pointermove", onMouseMove);
+    };
+  }, [startDrag, endDrag, initX, initY]);
+
+  const onPointerDown = useCallback(
+    (event: React.PointerEvent<HTMLDivElement>) => {
+      if (event.button === 0) {
+        startDrag(type, clip.media_id, clip.length);
+        currPosition.value = {
+          currX: event.pageX,
+          currY: event.pageY,
+        };
+        initPosition.value = {
+          initX: event.pageX,
+          initY: event.pageY,
+        };
+        canDrop.value = false;
+      }
+    },
+    [type, clip.media_id, startDrag, clip.length],
+  );
+
+  return (
+    <div className="relative h-16 w-16">
+      <div
+        id={`ani-clip-${clip.media_id}`}
+        className="absolute block h-16 w-16 rounded-lg bg-brand-secondary-700 p-2"
+        style={{ top: 0, left: 0 }}
+        onPointerDown={onPointerDown}
+      >
+        {clip.name}
+      </div>
+    </div>
+  );
+};

app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx

@@ -0,0 +1,25 @@
+import { useContext, useEffect, useState } from "react";
+import { AnimationElement } from "./AnimationElement";
+import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
+import { useSignals } from "@preact/signals-react/runtime";
+import { timelineHeight } from "~/pages/PageEnigma/store";
+
+export const AudioElements = () => {
+  const { audioClips } = useContext(TrackContext);
+  useSignals();
+  const [height, setHeight] = useState(0);
+
+  useEffect(() => {
+    setHeight(window.outerHeight - timelineHeight.value);
+  }, []);
+
+  return (
+    <div className="flex flex-wrap gap-3 overflow-y-auto" style={{ height }}>
+      {audioClips.map((clip) => {
+        return (
+          <AnimationElement key={clip.media_id} clip={clip} type="audio" />
+        );
+      })}
+    </div>
+  );
+};

app/pages/PageEnigma/comps/SidePanelTabs/index.tsx

@@ -1,6 +1,7 @@
 import { Tabs } from "~/modules/Tabs";
-import { TabStylization} from "./TabStylization";
-import { ClipElements } from "./ClipElements";
+import { TabStylization } from "./TabStylization";
+import { AnimationElements } from "./AnimationElements";
+import { AudioElements } from "~/pages/PageEnigma/comps/SidePanelTabs/AudioElements";
 
 export const SidePanelTabs = () => {
   return (
@@ -8,7 +9,7 @@ export const SidePanelTabs = () => {
       tabs={[
         {
           header: "Animation",
-          children: <ClipElements />,
+          children: <AnimationElements />,
         },
         {
           header: "Camera",
@@ -16,7 +17,7 @@ export const SidePanelTabs = () => {
         },
         {
           header: "Audio",
-          children: <p>Audio Tab</p>,
+          children: <AudioElements />,
         },
         {
           header: "Stylization",

app/pages/PageEnigma/comps/Timeline/Camera.tsx

@@ -1,7 +1,9 @@
 import { cameraGroup, fullWidth, updateCamera } from "~/pages/PageEnigma/store";
 import { TrackKeyFrames } from "~/pages/PageEnigma/comps/Timeline/TrackKeyFrames";
+import { useSignals } from "@preact/signals-react/runtime";
 
 export const Camera = () => {
+  useSignals();
   const { keyframes } = cameraGroup.value!;
 
   return (

app/pages/PageEnigma/comps/Timeline/ObjectTrack.tsx

@@ -15,7 +15,7 @@ export const ObjectTrack = () => {
           <TrackKeyFrames
             id={object.object_uuid}
             keyframes={object.keyframes}
-            title="Mask Position/Rotation"
+            title={`${object.name} Position/Rotation`}
             updateKeyframe={updateObject}
             style="objects"
           />

app/pages/PageEnigma/comps/Timeline/Timeline.tsx

@@ -8,6 +8,7 @@ import {
   timelineHeight,
   deleteAudioClip,
   deleteCharacterClip,
+  selectedItem,
 } from "~/pages/PageEnigma/store";
 import { useQueueHandler } from "~/pages/PageEnigma/comps/Timeline/utils/useQueueHandler";
 import { useSignals } from "@preact/signals-react/runtime";
@@ -17,7 +18,6 @@ import { Characters } from "~/pages/PageEnigma/comps/Timeline/Characters";
 import { ObjectGroups } from "~/pages/PageEnigma/comps/Timeline/ObjectGroups";
 import useUpdateKeyframe from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateKeyframe";
 import { Clip, Keyframe } from "~/pages/PageEnigma/models/track";
-import { selectedItem } from "~/pages/PageEnigma/store/selectedItem";
 
 function getItemType(item: Clip | Keyframe | null) {
   if (!item) {
@@ -50,9 +50,12 @@ export const Timeline = () => {
   }, []);
 
   const onDelete = useCallback(() => {
-    deleteCharacterClip(selectedItem.value as Clip);
-    deleteAudioClip(selectedItem.value as Clip);
-    deleteKeyframe(selectedItem.value as Keyframe);
+    if ((selectedItem.value as Clip).clip_uuid) {
+      deleteCharacterClip(selectedItem.value as Clip);
+      deleteAudioClip(selectedItem.value as Clip);
+    } else {
+      deleteKeyframe(selectedItem.value as Keyframe);
+    }
     selectedItem.value = null;
   }, [deleteKeyframe]);
 

app/pages/PageEnigma/comps/Timeline/TrackClip.tsx

@@ -1,8 +1,7 @@
 import { useMouseEventsClip } from "./utils/useMouseEventsClip";
 import { useState } from "react";
-import { scale } from "~/pages/PageEnigma/store";
+import { scale, selectedItem } from "~/pages/PageEnigma/store";
 import { Clip } from "~/pages/PageEnigma/models/track";
-import { selectedItem } from "~/pages/PageEnigma/store/selectedItem";
 
 interface Props {
   min: number;

app/pages/PageEnigma/comps/Timeline/TrackClips.tsx

@@ -37,15 +37,29 @@ export const TrackClips = ({
   const trackType = type ?? style;
 
   function onPointerOver() {
-    if (dragType.value !== trackType) {
-      return;
+    if (dragType.value === "animations") {
+      if (dragType.value !== trackType) {
+        return;
+      }
+    }
+    if (dragType.value === "audio") {
+      if (["lipSync", "audio"].indexOf(trackType) === -1) {
+        return;
+      }
     }
     dropId.value = id;
   }
 
   function onPointerMove(event: PointerEvent<HTMLDivElement>) {
-    if (dragType.value !== trackType) {
-      return;
+    if (dragType.value === "animations") {
+      if (dragType.value !== trackType) {
+        return;
+      }
+    }
+    if (dragType.value === "audio") {
+      if (["lipSync", "audio"].indexOf(trackType) === -1) {
+        return;
+      }
     }
     const track = document.getElementById(`track-${trackType}-${id}`);
     if (!track) {

app/pages/PageEnigma/comps/Timeline/TrackKeyFrame.tsx

@@ -1,15 +1,13 @@
 import { Keyframe } from "~/pages/PageEnigma/models/track";
 import { useState } from "react";
-import { scale } from "~/pages/PageEnigma/store";
+import { scale, selectedItem } from "~/pages/PageEnigma/store";
 import { useMouseEventsKeyframe } from "~/pages/PageEnigma/comps/Timeline/utils/useMouseEventsKeyframe";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faSortUpDown } from "@fortawesome/pro-solid-svg-icons";
-import { selectedItem } from "~/pages/PageEnigma/store/selectedItem";
 
 interface Props {
   min: number;
   max: number;
-  style: "character" | "camera" | "audio" | "objects";
   keyframe: Keyframe;
   updateKeyframe: (options: { id: string; offset: number }) => void;
 }
@@ -18,7 +16,6 @@ export const TrackKeyFrame = ({
   keyframe,
   min,
   max,
-  style,
   updateKeyframe,
 }: Props) => {
   const [offset, setOffset] = useState(keyframe.offset);
@@ -33,32 +30,19 @@ export const TrackKeyFrame = ({
   const selectedKeyframeId =
     (selectedItem.value as Keyframe | null)?.keyframe_uuid ?? "";
 
-  const classes = [
-    "absolute",
-    keyframe.keyframe_uuid === selectedKeyframeId
-      ? `bg-${style}-selected`
-      : `bg-${style}-clip`,
-  ];
-
   return (
     <button
-      className={[
-        ...classes,
-        "rounded-l-lg",
-        "block h-full",
-        keyframe.keyframe_uuid === selectedKeyframeId
-          ? "border-2 border-white focus-visible:outline-0"
-          : "",
-      ].join(" ")}
+      className={["block rotate-45 cursor-ew-resize", "absolute"].join(" ")}
       style={{
-        width: 15,
-        left: offset * 4 * scale.value,
-        cursor: "w-resize",
+        width: 18,
+        height: 18,
+        backgroundColor:
+          keyframe.keyframe_uuid === selectedKeyframeId ? "#FFDE67" : "#EEEEEE",
+        left: offset * 4 * scale.value - 6,
+        top: 8,
       }}
-      onPointerDown={(event) => onPointerDown(event, "left")}
+      onPointerDown={(event) => onPointerDown(event, "drag")}
       onClick={() => (selectedItem.value = keyframe)}
-    >
-      <FontAwesomeIcon icon={faSortUpDown} />
-    </button>
+    />
   );
 };

app/pages/PageEnigma/comps/Timeline/TrackKeyFrames.tsx

@@ -34,7 +34,6 @@ export const TrackKeyFrames = ({
                 ? keyframes[index + 1].offset
                 : filmLength.value * 60
             }
-            style={style}
             updateKeyframe={updateKeyframe}
             keyframe={keyframe}
           />

app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx

@@ -3,7 +3,6 @@ import {
   MediaClip,
   Keyframe,
   QueueKeyframe,
-  Clip,
 } from "~/pages/PageEnigma/models/track";
 
 export const TrackContext = createContext<{
@@ -16,11 +15,7 @@ export const TrackContext = createContext<{
   audioClips: MediaClip[];
 
   // drag and drop
-  startDrag: (
-    type: "animations" | "lipSync",
-    id: string,
-    length: number,
-  ) => void;
+  startDrag: (type: "animations" | "audio", id: string, length: number) => void;
   endDrag: () => void;
 }>({
   addKeyframe: () => {},

app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx

@@ -1,14 +1,11 @@
 import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
 import { ReactNode, useCallback, useEffect, useMemo, useState } from "react";
-import {
-  ClipType,
-  Keyframe,
-  Clip,
-  MediaClip,
-} from "~/pages/PageEnigma/models/track";
+import { ClipType, MediaClip } from "~/pages/PageEnigma/models/track";
 import useUpdateDragDrop from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop";
 import {
   addCharacterAnimation,
+  addCharacterAudio,
+  addGlobalAudio,
   canDrop,
   dragId,
   dragType,
@@ -59,6 +56,20 @@ export const TrackProvider = ({ children }: Props) => {
           offset: dropOffset.value,
         });
       }
+      if (dragType.value === "audio") {
+        addCharacterAudio({
+          clipId: dragId.value!,
+          characterId: dropId.value,
+          audioClips,
+          offset: dropOffset.value,
+        });
+        addGlobalAudio({
+          clipId: dragId.value!,
+          audioId: dropId.value,
+          audioClips,
+          offset: dropOffset.value,
+        });
+      }
     }
     endDrag();
   }, [animationClips, endDrag]);

app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateDragDrop.ts

@@ -8,7 +8,7 @@ import {
 
 export default function useUpdateDragDrop() {
   const startDrag = useCallback(
-    (type: "animations" | "lipSync", id: string, length: number) => {
+    (type: "animations" | "audio", id: string, length: number) => {
       dragId.value = id;
       dragType.value = type;
       clipLength.value = length;

app/pages/PageEnigma/js/editor.ts

@@ -12,7 +12,7 @@ import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js";
 import { SAOPass } from "three/addons/postprocessing/SAOPass.js";
 import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js";
 import { BokehPass } from "three/addons/postprocessing/BokehPass.js";
-import { FFmpeg, createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg";
+import { createFFmpeg, fetchFile, FFmpeg } from "@ffmpeg/ffmpeg";
 import AudioEngine from "./audio_engine.js";
 import TransformEngine from "./transform_engine.js";
 import { TimeLine, TimelineDataState } from "./timeline.js";
@@ -22,6 +22,7 @@ import { AnimationEngine } from "./animation_engine.js";
 
 import { PointerLockControls } from "three/addons/controls/PointerLockControls.js";
 import { APPUI_ACTION_TYPES } from "../reducers";
+import { ClipGroup } from "~/pages/PageEnigma/models/track";
 
 class EditorState {
   // {
@@ -1144,7 +1145,10 @@ class Editor {
     this.dispatchAppUiState({
       type: APPUI_ACTION_TYPES.SHOW_CONTROLPANELS_SCENEOBJECT,
       payload: {
-        group: "object", // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
+        group:
+          this.selected.name === "::CAM::"
+            ? ClipGroup.CAMERA
+            : ClipGroup.OBJECT, // TODO: add meta data to determine what it is a camera or a object or a character into prefab clips
         object_uuid: this.selected.uuid,
         object_name: this.selected.name,
         version: this.version,

app/pages/PageEnigma/store/audioGroup.ts

@@ -48,12 +48,22 @@ export function updateAudio({
   };
 }
 
-export function addGlobalAudio(
-  dragId: string,
-  audioClips: MediaClip[],
-  offset: number,
-) {
-  const clip = audioClips.find((row) => row.media_id === dragId);
+export function addGlobalAudio({
+  clipId,
+  audioId,
+  audioClips,
+  offset,
+}: {
+  clipId: string;
+  audioId: string;
+  audioClips: MediaClip[];
+  offset: number;
+}) {
+  if (audioGroup.value.id !== audioId) {
+    return;
+  }
+
+  const clip = audioClips.find((row) => row.media_id === clipId);
   if (!clip) {
     return;
   }

app/pages/PageEnigma/store/cameraGroup.ts

@@ -33,6 +33,7 @@ export function updateCamera({ id, offset }: { id: string; offset: number }) {
 }
 
 export function addCameraKeyframe(keyframe: QueueKeyframe, offset: number) {
+  console.log("cam", keyframe);
   const newKeyframe = {
     version: keyframe.version,
     keyframe_uuid: uuid.v4(),

app/pages/PageEnigma/store/characterGroups.ts

@@ -81,7 +81,7 @@ export function updateCharacters({
 
       return {
         ...character,
-        positionClips: newPositionKeyframes,
+        positionKeyframes: newPositionKeyframes,
       };
     });
   }

app/pages/PageEnigma/store/dragAndDrop.ts

@@ -1,6 +1,6 @@
 import { signal } from "@preact/signals-core";
 
-export const dragType = signal<"animations" | "lipSync" | null>(null);
+export const dragType = signal<"animations" | "audio" | null>(null);
 export const dragId = signal<string | null>(null);
 export const canDrop = signal(false);
 export const overTimeline = signal(false);

app/pages/PageEnigma/store/index.ts

@@ -3,5 +3,6 @@ export * from "./cameraGroup";
 export * from "./characterGroups";
 export * from "./dragAndDrop";
 export * from "./objectGroup";
+export * from "./selectedItem";
 export * from "./sizing";
 export * from "./timing";

app/pages/PageEnigma/store/objectGroup.ts

@@ -21,10 +21,27 @@ export function updateObject({ id, offset }: { id: string; offset: number }) {
     objects: oldObject.objects.map((object) => ({
       object_uuid: object.object_uuid,
       name: object.name,
-      keyframes: object.keyframes.map((keyFrame) => ({
-        ...keyFrame,
-        offset: keyFrame.keyframe_uuid === id ? offset : keyFrame.offset,
-      })),
+      keyframes: object.keyframes.map((keyFrame) => {
+        if (keyFrame.keyframe_uuid !== id) {
+          return {
+            ...keyFrame,
+          };
+        }
+
+        Queue.publish({
+          queueName: QueueNames.TO_ENGINE,
+          action: toEngineActions.UPDATE_KEYFRAME,
+          data: {
+            ...keyFrame,
+            offset,
+          },
+        });
+
+        return {
+          ...keyFrame,
+          offset,
+        };
+      }),
     })),
   };
 }
@@ -78,7 +95,7 @@ export function deleteObjectKeyframe(keyframe: Keyframe) {
   const obj = oldObjectGroup.objects.find(
     (row) => row.object_uuid === keyframe.object_uuid,
   );
-  console.log("delete", obj);
+  console.log("delete", obj, keyframe.keyframe_uuid);
   if (!obj) {
     return oldObjectGroup;
   }
@@ -98,7 +115,7 @@ export function deleteObjectKeyframe(keyframe: Keyframe) {
   ];
 
   if (newKeyframes.length) {
-    return {
+    objectGroup.value = {
       ...oldObjectGroup,
       objects: [
         ...oldObjectGroup.objects.map((object) => ({
@@ -109,6 +126,7 @@ export function deleteObjectKeyframe(keyframe: Keyframe) {
         })),
       ],
     };
+    return;
   }
   objectGroup.value = {
     ...oldObjectGroup,