Merge branch 'test' into styling/asset-drawer

app/pages/PageEnigma/PageEnigma.tsx

@@ -22,6 +22,7 @@ export const PageEnigma = () => {
   useSignals();
   useEffect(() => {
     function setPage() {
+      // TODO address this issue with zooming
       pageHeight.value = window.innerHeight;
       pageWidth.value = window.outerWidth;
     }

app/pages/PageEnigma/PageEnigmaComponent.tsx

@@ -53,7 +53,6 @@ export const PageEnigmaComponent = () => {
   const width = sidePanelVisible.value
     ? pageWidth.value - dndWidth - 66
     : pageWidth.value - 66;
-
   const height =
     dndTimelineHeight.value > -1
       ? pageHeight.value - dndTimelineHeight.value - 68

app/pages/PageEnigma/Queue/Queue.ts

@@ -7,19 +7,20 @@ import {
 } from "~/pages/PageEnigma/models";
 import { toTimelineActions } from "./toTimelineActions";
 import { ClipUI } from "../datastructures/clips/clip_ui";
+import { MediaItem } from "~/pages/PageEnigma/models";
 class Queue {
   private _queue: Record<
     string,
     {
       action: fromEngineActions | toEngineActions | toTimelineActions;
-      data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[];
+      data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[] | MediaItem;
     }[]
   > = {};
   private _subscribers: Record<
     string,
     (entry: {
       action: fromEngineActions | toEngineActions | toTimelineActions;
-      data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[];
+      data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[] | MediaItem;
     }) => void
   > = {};
 
@@ -30,7 +31,7 @@ class Queue {
   }: {
     queueName: string;
     action: fromEngineActions | toEngineActions | toTimelineActions;
-    data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[] ;
+    data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[] | MediaItem ;
   }) {
     if (!this._queue[queueName]) {
       this._queue[queueName] = [];
@@ -47,7 +48,7 @@ class Queue {
     queueName: string,
     onMessage: (entry: {
       action: fromEngineActions | toEngineActions | toTimelineActions;
-      data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[];
+      data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[] | MediaItem;
     }) => void,
   ) {
     this._subscribers[queueName] = onMessage;

app/pages/PageEnigma/Queue/fromEngineActions.ts

@@ -1,5 +1,6 @@
 export enum fromEngineActions {
   UPDATE_TIME = "update_time",
   UPDATE_TIME_LINE = "update_timeline",
+  UPDATE_CHARACTER_ID = "update_character_id",
   // data={currentTime: number}
 }

app/pages/PageEnigma/Queue/toEngineActions.ts

@@ -2,6 +2,11 @@ export enum toEngineActions {
   ADD_CLIP = "add_clip",
   // data=QueueClip
   ADD_KEYFRAME = "add_keyframe",
+
+  ADD_CHARACTER = "add_character",
+
+  ADD_OBJECT = "add_object",
+
   // data= QueueKeyframe
   DELETE_CLIP = "delete_clip",
   // data=QueueClip

app/pages/PageEnigma/comps/Timeline/utils/useQueueHandler.ts

@@ -1,26 +1,43 @@
 import { fromEngineActions } from "~/pages/PageEnigma/Queue/fromEngineActions";
 import { useSignals } from "@preact/signals-react/runtime";
 import { useCallback, useContext, useEffect } from "react";
-import { currentTime } from "~/pages/PageEnigma/store";
+import {
+  addNewCharacter,
+  currentTime,
+  loadAudioData,
+  loadCameraData,
+  loadCharacterData,
+  loadObjectData,
+} from "~/pages/PageEnigma/store";
 import Queue from "~/pages/PageEnigma/Queue/Queue";
 import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
 import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
 import {
+  ClipGroup,
+  MediaItem,
   QueueClip,
   QueueKeyframe,
   UpdateTime,
 } from "~/pages/PageEnigma/models";
 import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
 import { toTimelineActions } from "~/pages/PageEnigma/Queue/toTimelineActions";
+import { ClipUI } from "~/pages/PageEnigma/datastructures/clips/clip_ui";
 
 interface Arguments {
   action: fromEngineActions | toEngineActions | toTimelineActions;
-  data: QueueClip | UpdateTime | QueueKeyframe;
+  data: QueueClip | UpdateTime | QueueKeyframe | ClipUI[] | MediaItem;
 }
 
+const LOADING_FUNCTIONS: Record<ClipGroup, (item: ClipUI) => void> = {
+  [ClipGroup.CHARACTER]: loadCharacterData,
+  [ClipGroup.OBJECT]: loadObjectData,
+  [ClipGroup.CAMERA]: loadCameraData,
+  [ClipGroup.GLOBAL_AUDIO]: loadAudioData,
+};
+
 export function useQueueHandler() {
   useSignals();
-  const { addKeyframe } = useContext(TrackContext);
+  const { addKeyframe, clearExistingData } = useContext(TrackContext);
 
   const handleFromEngineActions = useCallback(({ action, data }: Arguments) => {
     console.log("FROM ENGINE", action, data);
@@ -29,22 +46,33 @@ export function useQueueHandler() {
         currentTime.value = (data as UpdateTime).currentTime;
         break;
       case fromEngineActions.UPDATE_TIME_LINE:
-        console.log(action)
+        clearExistingData();
+        (data as ClipUI[]).forEach((item) => {
+          LOADING_FUNCTIONS[item.group](item);
+        });
+        break;
+      case fromEngineActions.UPDATE_CHARACTER_ID: {
+        addNewCharacter(data as MediaItem);
+        break;
+      }
       default:
         throw new Error(`Unknown action ${action}`);
     }
   }, []);
 
-  const handleToTimelineActions = useCallback(({ action, data }: Arguments) => {
-    console.log("TO TIMELINE", action, data);
-    switch (action) {
-      case toTimelineActions.ADD_KEYFRAME:
-        addKeyframe(data as QueueKeyframe, currentTime.value);
-        break;
-      default:
-        throw new Error(`Unknown action ${action}`);
-    }
-  }, []);
+  const handleToTimelineActions = useCallback(
+    ({ action, data }: Arguments) => {
+      console.log("TO TIMELINE", action, data);
+      switch (action) {
+        case toTimelineActions.ADD_KEYFRAME:
+          addKeyframe(data as QueueKeyframe, currentTime.value);
+          break;
+        default:
+          throw new Error(`Unknown action ${action}`);
+      }
+    },
+    [addKeyframe],
+  );
 
   useEffect(() => {
     Queue.subscribe(QueueNames.FROM_ENGINE, handleFromEngineActions);

app/pages/PageEnigma/contexts/TrackContext/TrackContext.tsx

@@ -6,6 +6,9 @@ export const TrackContext = createContext<{
   addKeyframe: (keyframe: QueueKeyframe, offset: number) => void;
   deleteKeyframe: (keyframe: Keyframe) => void;
 
+  // misc
+  clearExistingData: () => void;
+
   // drag and drop
   startDrag: (item: MediaItem) => void;
   endDrag: () => void;
@@ -13,6 +16,8 @@ export const TrackContext = createContext<{
   addKeyframe: () => {},
   deleteKeyframe: () => {},
 
+  clearExistingData: () => {},
+
   startDrag: () => {},
   endDrag: () => {},
 });

app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx

@@ -6,10 +6,16 @@ import {
   addCharacterAnimation,
   addCharacterAudio,
   addGlobalAudio,
+  addCharacter,
   canDrop,
   dragItem,
   dropId,
   dropOffset,
+  addObject,
+  characterGroups,
+  cameraGroup,
+  audioGroup,
+  objectGroup,
 } from "~/pages/PageEnigma/store";
 import useUpdateKeyframe from "~/pages/PageEnigma/contexts/TrackContext/utils/useUpdateKeyframe";
 
@@ -24,52 +30,75 @@ export const TrackProvider = ({ children }: Props) => {
 
   // cross group functions
   const dropClip = useCallback(() => {
+    if (dragItem.value) {
+      const mediaItem = dragItem.value;
+      if (mediaItem.type === AssetType.CHARACTER) {
+        addCharacter(dragItem.value);
+      }
+      // if (dragItem.value.type === AssetType.CAMERA) {
+      //   console.log("Dragged In Camera Type")
+      // }
+      if (dragItem.value.type === AssetType.OBJECT) {
+        addObject(dragItem.value);
+      }
+    }
+    // if (dragItem.value.type === AssetType.SHAPE) {
+    //   console.log("Dragged In Shape Type")
+    // }
 
-    console.log(`${canDrop.value} ${dragItem.value} ${dragItem.value.type}`)
     if (canDrop.value && dragItem.value) {
       if (dragItem.value.type === AssetType.ANIMATION) {
         addCharacterAnimation({
-          dragItem: dragItem.value!,
+          dragItem: dragItem.value,
           characterId: dropId.value,
           offset: dropOffset.value,
         });
       }
       if (dragItem.value.type === AssetType.AUDIO) {
+        console.log("add audio", dropId.value);
         addCharacterAudio({
-          dragItem: dragItem.value!,
+          dragItem: dragItem.value,
           characterId: dropId.value,
           offset: dropOffset.value,
         });
         addGlobalAudio({
-          dragItem: dragItem.value!,
+          dragItem: dragItem.value,
           audioId: dropId.value,
           offset: dropOffset.value,
         });
       }
-      if (dragItem.value.type === AssetType.CHARACTER) {
-        console.log("Dragged In Character Type")
-      }
-      if (dragItem.value.type === AssetType.CAMERA) {
-        console.log("Dragged In Camera Type")
-      }
-      if (dragItem.value.type === AssetType.OBJECT) {
-        console.log("Dragged In Object Type")
-      }
-      if (dragItem.value.type === AssetType.SHAPE) {
-        console.log("Dragged In Shape Type")
-      }
     }
     endDrag();
   }, [endDrag]);
 
+  const clearExistingData = useCallback(() => {
+    characterGroups.value = [];
+    cameraGroup.value = {
+      id: "CG1",
+      keyframes: [],
+    };
+    audioGroup.value = {
+      id: "AG-1",
+      clips: [],
+      muted: false,
+    };
+    objectGroup.value = {
+      id: "OG1",
+      objects: [],
+    };
+  }, []);
+
   const values = useMemo(() => {
     return {
       ...keyframes,
 
+      clearExistingData,
+
       ...dragDrop,
       endDrag: dropClip,
     };
-  }, [keyframes, dragDrop, dropClip]);
+  }, [keyframes, dragDrop, dropClip, clearExistingData]);
+
   return (
     <TrackContext.Provider value={values}>{children}</TrackContext.Provider>
   );

app/pages/PageEnigma/contexts/TrackContext/utils/useUpdateKeyframe.ts

@@ -28,6 +28,7 @@ const DELETE_KEYFRAME: Record<ClipGroup, (keyframe: Keyframe) => void> = {
 
 export default function useUpdateKeyframe() {
   const addKeyframe = useCallback((keyframe: QueueKeyframe, offset: number) => {
+    console.log("k", keyframe);
     ADD_KEYFRAME[keyframe.group](keyframe, offset);
   }, []);
 

app/pages/PageEnigma/datastructures/clips/clip_ui.ts

@@ -9,9 +9,12 @@ export class ClipUI {
   name: string;
   media_id: string;
   object_uuid: string;
+  object_name: string;
   offset: number; // in frames
   length: number; // in frames
   should_play: boolean;
+  clip_uuid: string;
+  keyframe_offset: number;
 
   constructor(
     version: number,
@@ -19,21 +22,28 @@ export class ClipUI {
     group: ClipGroup,
     name: string,
     media_id: string,
+    clip_uuid: string,
     object_uuid: string,
+    object_name: string,
     offset: number,
     length: number,
+    keyframe_offset: number = 0,
   ) {
     this.version = version;
     this.group = group; // Only needed for UI
 
+    this.clip_uuid = clip_uuid;
+
     this.name = name; // UI
     this.type = type; // UI and Animation / Audio / Lipsync /  : Engine
 
     this.object_uuid = object_uuid; // Animation / Audio / Lipsync /  : Engine
+    this.object_name = object_name;
     this.media_id = media_id; //  Animation / Audio / Lipsync /  : Engine
     this.offset = offset;
     this.length = length;
     this.should_play = true;
+    this.keyframe_offset = keyframe_offset;
   }
 
   toJSON(): any {
@@ -42,10 +52,13 @@ export class ClipUI {
       group: this.group,
       name: this.name,
       type: this.type,
+      clip_uuid: this.clip_uuid,
       object_uuid: this.object_uuid,
+      object_name: this.object_name,
       media_id: this.media_id,
       start_offset: this.offset,
       ending_offset: this.length,
+      keyframe_offset: this.keyframe_offset,
     };
   }
 }

app/pages/PageEnigma/js/editor.ts

@@ -311,7 +311,7 @@ class Editor {
 
     this.timeline.scene = this.activeScene;
 
-    this._test_demo();
+    //this._test_demo();
 
     this.renderer.domElement.addEventListener(
       "mousedown",
@@ -1076,7 +1076,6 @@ class Editor {
     this.raycaster.setFromCamera(this.mouse, this.camera);
     const interactable: any[] = [];
     this.activeScene.scene.children.forEach((child: THREE.Object3D) => {
-      // console.log(child);
       if (child.name != "") {
         if (child.type == "Mesh" || child.type == "Object3D" || child.type == "Group") {
           interactable.push(child);

app/pages/PageEnigma/js/timeline.ts

@@ -42,6 +42,7 @@ export class TimeLine {
     animation_engine: AnimationEngine;
     // lip sync engine
     lipSync_engine: LipSyncEngine;
+    characters: {[key: string]: string};
 
     scene: Scene;
 
@@ -55,6 +56,7 @@ export class TimeLine {
         scene: Scene,
     ) {
         this.timeline_items = [];
+        this.characters = {};
         this.absolute_end = 60 * 12;
         this.timeline_limit = 0; // 5 seconds
 
@@ -124,11 +126,47 @@ export class TimeLine {
             case toEngineActions.UNMUTE:
                 await this.mute(data, true);
                 break;
+            case toEngineActions.ADD_CHARACTER:
+                await this.addCharacter(data);
+                break;
+            case toEngineActions.ADD_OBJECT:
+                await this.addObject(data);
+                break;
             default:
                 console.log("Action Not Wired", action);
         }
     }
 
+    public async addCharacter(data: any) {
+        let media_id = data.data['media_id'];
+        let name = data.data['name'];
+        let type = data.data['type'];
+
+        let obj = await this.scene.load_glb(media_id);
+        obj.userData['name'] = name;
+        obj.name = name;
+        let object_uuid = obj.uuid;
+
+        console.log("data.data", data.data)
+
+        this.characters[object_uuid] = type;
+
+        data.data['object_uuid'] = object_uuid;
+        Queue.publish({
+            queueName: QueueNames.FROM_ENGINE,
+            action: fromEngineActions.UPDATE_CHARACTER_ID,
+            data: data.data,
+        });
+    }
+
+    public async addObject(data: any) {
+        let media_id = data.data['media_id'];
+        let name = data.data['name'];
+        let obj = await this.scene.load_glb(media_id);
+        obj.userData['name'] = name;
+        obj.name = name;
+    }
+
     public async addKeyFrame(data: any) {
         // KeyFrame Object
         // version: number;
@@ -142,12 +180,21 @@ export class TimeLine {
         // selected?: boolean;
         let data_json = data['data'];
         let uuid = data_json['object_uuid'];
+        let keyframe_uuid = data_json['keyframe_uuid'];
 
         let object_name = this.scene.get_object_by_uuid(uuid)?.name;
         if (object_name == undefined) {
             object_name = "undefined"
         }
 
+        for (let key in this.characters) {
+            let element = this.characters[key];
+            if (key == uuid) {
+                data_json['group'] = element;
+                break;
+            }
+        }
+
         let new_item = this.transform_engine.addFrame(uuid,
             this.absolute_end,
             data_json['position'],
@@ -162,9 +209,12 @@ export class TimeLine {
                 data_json['group'],
                 object_name,
                 "",
+                keyframe_uuid,
                 uuid,
+                object_name,
                 0,
-                this.absolute_end))
+                this.absolute_end,
+                data_json['offset']))
         }
 
         this.scene.createPoint(data_json['position'], data_json['keyframe_uuid']);
@@ -184,6 +234,13 @@ export class TimeLine {
         const type = data["data"]["type"];
         const offset = data["data"]["offset"];
         const end_offset = data["data"]["length"] + offset;
+        let object_name = this.scene.get_object_by_uuid(object_uuid)?.name;
+        const clip_uuid = data["data"]['clip_uuid'];
+        console.log(data);
+
+        if(object_name == undefined) {
+            object_name = "Undefined."
+        }
 
         switch (type) {
             case "animation":
@@ -199,11 +256,13 @@ export class TimeLine {
                     this.addPlayableClip(
                         new ClipUI(
                             version,
-                            "lipsync",// TODO potiential bug ..
+                            "lipsync", // TODO potiential bug ..
                             group,
                             name,
                             media_id,
+                            clip_uuid,
                             object_uuid,
+                            object_name,
                             offset,
                             end_offset));
                     return;
@@ -221,7 +280,9 @@ export class TimeLine {
                 group,
                 name,
                 media_id,
+                clip_uuid,
                 object_uuid,
+                object_name,
                 offset,
                 end_offset,
             ),
@@ -286,6 +347,7 @@ export class TimeLine {
     }
 
     public async addPlayableClip(clip: ClipUI): Promise<void> {
+        console.log("clip", clip)
         this.timeline_items.push(clip);
     }
 
@@ -449,96 +511,3 @@ export class TimeLine {
     }
 }
 
-// How much timeline precision we have
-//const percision = 100 // using a fake update loop mock
-//
-//class AudioEngineMock {
-//    async play(media_id:string) {
-//        console.log("Audio Playing {media_id}")
-//    }
-//}
-//
-//// Visual verification tests.
-//function CheckIfBasicAudioClipWorks() {
-//    const timeline = new TimeLine()
-//    timeline.addPlayableClip(new ClipUI(1.0,'audio',1,0))
-//    timeline.play()
-//}
-//
-//function CheckIfBasicTransformClipWorks() {
-//    const timeline = new TimeLine()
-//    timeline.addPlayableClip(new ClipUI(1.0,'transform',2,0))
-//    timeline.play()
-//}
-//
-//function CheckIfBasicClipWorks() {
-//    const timeline = new TimeLine()
-//    timeline.addPlayableClip(new ClipUI(1.0,'animation',3,0))
-//    timeline.play()
-//}
-
-// function CheckIfBasicClipWorks3SecondsAfterTimelineStops() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new ClipUI("clip1", 0, 10000))
-//     timeline.play()
-// }
-
-// function CheckIfTwoClipsAtTheSameTimeWorks() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new ClipUI("clip1", 0, 1000))
-//     timeline.play()
-// }
-
-// function CheckIfTwoClipsOneAfterAnotherWorks() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new ClipUI("clip1", 0, 1000))
-//     timeline.addPlayableClip(new ClipUI("clip2", 0, 2000))
-//     timeline.play()
-// }
-
-// function CheckIfTimeLineStopBeforeClipPlays() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new ClipUI("clip3",0,1000))
-//     timeline.play()
-//     setInterval(async ()=> {
-//         timeline.stop()
-//     },1100)
-//     console.log("Stopped")
-// }
-
-// function CheckIfTimeLineStartAfterClipPlays() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new ClipUI("clip3",0,1000))
-//     timeline.play()
-//     setInterval(async ()=> {
-//         timeline.stop()
-//     },500)
-//     console.log("Stopped")
-// }
-
-// function CheckIfClipsPlayAllTogetherConcurrently() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new ClipUI("clip1",0,1000))
-//     timeline.addPlayableClip(new ClipUI("clip2",0,1000))
-//     timeline.addPlayableClip(new ClipUI("clip3",0,1000))
-//     timeline.addPlayableClip(new ClipUI("clip4",0,1000))
-//     timeline.play()
-// }
-
-// function CheckIfClipsPlayAllTogether() {
-//     const timeline = new TimeLine()
-//     timeline.addPlayableClip(new TrackClip("clip1",0,1000))
-//     timeline.addPlayableClip(new TrackClip("clip2",0,1000))
-//     timeline.addPlayableClip(new TrackClip("clip3",0,1000))
-//     timeline.addPlayableClip(new TrackClip("clip4",0,1000))
-//     timeline.play()
-// }
-
-//CheckIfBasicClipWorks()
-//CheckIfTwoClipsAtTheSameTimeWorks()
-// CheckIfTwoClipsOneAfterAnotherWorks()
-// CheckIfTimeLineStopBeforeClipPlays()
-// CheckIfTimeLineStartAfterClipPlays()
-// CheckIfClipsPlayAllTogetherConcurrently()
-
-// CheckIfClipsPlayAllTogether()

app/pages/PageEnigma/models/assets.ts

@@ -2,9 +2,10 @@ export interface MediaItem {
   version: number;
   type: AssetType;
   media_id: string;
+  object_uuid?: string;
   name: string;
   length?: number;
-  thumbnail: string;
+  thumbnail?: string;
   isMine?: boolean;
   isBookmarked?: boolean;
 }
@@ -16,7 +17,7 @@ export enum AssetType {
   ANIMATION = "animation",
   CAMERA = "camera",
   SHAPE = "shape",
-  STYLE = "style" // TODO Remove 
+  STYLE = "style", // TODO Remove
 }
 
 export enum AssetFilterOption {

app/pages/PageEnigma/models/track.ts

@@ -17,7 +17,6 @@ export enum ClipType {
   TRANSFORM = "transform",
   AUDIO = "audio",
   ANIMATION = "animation",
-  LIPSYNC = "lipsync"
 }
 
 export enum ClipGroup {
@@ -41,6 +40,7 @@ export interface Keyframe {
 
 export interface CharacterGroup {
   id: string;
+  name: string;
   muted: boolean;
   animationClips: Clip[];
   positionKeyframes: Keyframe[];

app/pages/PageEnigma/proxy/storyteller_proxy_timeline.ts

@@ -53,6 +53,8 @@ export class StoryTellerProxyTimeline {
             audio: await this.getItemsDict(this.audio_engine.clips),
             lipsync: await this.getItemsDict(this.lipsync_engine.clips),
         }
+        console.log(timeline_json)
+
         return timeline_json;
     }
 
@@ -64,8 +66,11 @@ export class StoryTellerProxyTimeline {
                 let clip = new ClipUI(
                     element.version, element.type, 
                     element.group, element.name, 
-                    element.media_id, element.object_uuid, 
-                    element.start_offset, element.ending_offset);
+                    element.media_id, element.clip_uuid,
+                    element.object_uuid, 
+                    element.object_name,
+                    element.start_offset, element.ending_offset,
+                    element.keyframe_offset);
                 new_clips.push(clip);
             }
         }
@@ -124,6 +129,7 @@ export class StoryTellerProxyTimeline {
         this.lipsync_engine.clips = await this.loadLipsyncClips(timeline['lipsync']);
         this.animation_engine.clips = await this.loadAnimationClips(timeline['animation']);
 
+        console.log(this.timeline.timeline_items)
 
         this.timeline.updateUI()
     }