Merge branch 'test' into feature/timeline

app/components/H4.tsx

@@ -0,0 +1,19 @@
+import { twMerge } from "tailwind-merge"
+
+export const H4 = ({
+  className,
+  children,
+}:{
+  className?: string,
+  children: React.ReactNode,
+})=>{
+  return(
+    <h4
+      className={
+        twMerge("text-base font-medium text-white", className)
+      }
+    >
+      {children}
+    </h4>
+  )
+}

app/components/ItemPicker.tsx

@@ -0,0 +1,17 @@
+import { H4 } from ".";
+
+interface ItemPickerProps extends React.ImgHTMLAttributes<HTMLImageElement> {
+  label: string
+}
+
+export const ItemPicker = ({
+  label,
+  ...imgProps
+}:ItemPickerProps) => {
+  return (
+    <div className="relative border border-ui-border rounded-lg overflow-hidden">
+      <img className="aspect-square min-w-20" {...imgProps} />
+      <H4 className="absolute bottom-0 left-2 drop-shadow-md">{label}</H4>
+    </div>
+  )
+}

app/components/Textarea.tsx

@@ -0,0 +1,17 @@
+import { TextareaHTMLAttributes } from "react"
+import { twMerge } from "tailwind-merge";
+
+export const Textarea = ({
+  className,
+  ...rest
+}:TextareaHTMLAttributes<HTMLTextAreaElement>) => {
+  return(
+    <textarea
+      {...rest}
+      className={twMerge(
+        "rounded-lg bg-ui-controls border border-ui-panel-border py-2 px-4"
+        ,className
+      )}
+    />
+  );
+}

app/components/index.tsx

@@ -3,4 +3,9 @@ import { ButtonIcon } from "./ButtonIcon";
 import { ButtonLink } from "./ButtonLink";
 import { ButtonIconSelect } from "./ButtonIconSelect";
 
-export { Button, ButtonIcon, ButtonLink, ButtonIconSelect };
+import {H4} from "./H4";
+
+import { ItemPicker } from "./ItemPicker";
+import { Textarea } from "./Textarea";
+
+export { Button, ButtonIcon, ButtonLink, ButtonIconSelect, H4, ItemPicker, Textarea};

app/contexts/EngineContext.tsx

@@ -0,0 +1,4 @@
+import { createContext } from "react";
+import Editor from "~/pages/PageEnigma/js/editor";
+
+export const EngineContext = createContext<Editor|null>(null);

app/contexts/EngineProvider.tsx

@@ -0,0 +1,34 @@
+import {
+  ReactNode,
+  useEffect,
+  useRef,
+} from "react";
+import { EngineContext } from "./EngineContext";
+import Editor from "~/pages/PageEnigma/js/editor";
+
+interface Props{
+  children:ReactNode;
+}
+
+export const EngineProvider = ({ children }: Props)=>{
+  const editorRef = useRef<Editor | null>(null);
+
+  useEffect(() => {
+    //componentDidMount
+    if (editorRef.current !== null ) {
+      console.warn('Editor Engine already exist');
+    }else if (document.getElementById("video-scene") === null){
+      console.error('Editor Engine need a target cavas with the id "video-scene"');
+    }else{
+      editorRef.current = new Editor();
+      editorRef.current.initialize();
+    }
+  }, []);
+
+  return(
+    <EngineContext.Provider value={editorRef.current}>
+      {children}
+    </EngineContext.Provider>
+  );
+}
+

app/modules/ButtonDialogue/ButtonDialogue.tsx

@@ -8,6 +8,7 @@ type UnionedButtonProps =  {label?:string } & ButtonPropsI;
 
 interface ButtonDialoguePropsI  {
   buttonProps?: UnionedButtonProps;
+  confirmButtonProps?: UnionedButtonProps;
   closeButtonProps?: UnionedButtonProps;
   title?: string;
   children: React.ReactNode;
@@ -15,6 +16,7 @@ interface ButtonDialoguePropsI  {
 
 export const ButtonDialogue = ({
   buttonProps : unionedButtonProps,
+  confirmButtonProps,
   closeButtonProps : unionedCloseButtonProps,
   title,
   children,
@@ -80,14 +82,30 @@ export const ButtonDialogue = ({
                     {children}
                   </div>
 
-                  <div className="mt-4 flex justify-end	">
-                  <Button
-                    type="button"
-                    onClick={closeModal}
-                    {...closeButtonProps}
-                  >
-                    {closeButtonLabel}
-                  </Button>
+                  <div className="mt-4 flex justify-between ">
+                    {confirmButtonProps &&
+                      <Button
+                        type="button"
+                        {...confirmButtonProps}
+                        onClick={(e)=>{
+                          if(confirmButtonProps.onClick){
+                            confirmButtonProps.onClick(e);
+                          }
+                          closeModal();
+                        }}
+                        
+                      >
+                        {confirmButtonProps.label ? confirmButtonProps.label : "Confirm" }
+                      </Button>
+                    }
+                    <span />
+                    <Button
+                      type="button"
+                      onClick={closeModal}
+                      {...closeButtonProps}
+                    >
+                      {closeButtonLabel}
+                    </Button>
                   </div>
                 </Dialog.Panel>
               </Transition.Child>

app/modules/SidePanel/SidePanel.tsx

@@ -7,7 +7,11 @@ import {
   faChevronRight,
 } from "@fortawesome/pro-solid-svg-icons";
 
-export const SidePanel = () => {
+interface Props {
+  children: React.ReactNode;
+}
+
+export const SidePanel = ({children}:Props) => {
   const [isVisible, setIsVisible] = useState(true);
 
   return (
@@ -27,27 +31,11 @@ export const SidePanel = () => {
         leaveFrom="translate-x-0"
         leaveTo="translate-x-full"
       >
-        <Tabs
-          tabs={[
-            {
-              header: "Animation",
-              children: <p>Animation Tab</p>,
-            },
-            {
-              header: "Camera",
-              children: <p>Camera Tab</p>,
-            },
-            {
-              header: "Audio",
-              children: <p>Audio Tab</p>,
-            },
-            {
-              header: "Styling",
-              children: <p>Styling Tab</p>,
-            },
-          ]}
-        />
+
+          {children}
+
       </Transition>
     </div>
+
   );
 };

app/pages/PageEnigma/PageEnigma.tsx

@@ -1,52 +1,27 @@
 import { useCallback, useEffect, useRef, useState } from "react";
 
-import { faWandSparkles } from "@fortawesome/pro-solid-svg-icons";
+import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
 
 import { Button, ButtonLink } from "~/components";
-import { ButtonDialogue } from "~/modules/ButtonDialogue";
+// import { ButtonDialogue } from "~/modules/ButtonDialogue";
 import { TopBarHelmet } from "~/modules/TopBarHelmet/TopBarHelmet";
 import { SidePanel } from "~/modules/SidePanel";
-import { Tabs } from "~/modules/Tabs";
 import { Controls3D } from "./comps/Controls3D";
+import { ControlsTopButtons } from "./comps/ControlsTopButtons";
 import { ControlsVideo } from "./comps/ControlsVideo";
+import { PreviewWindow } from "./comps/PreviewWindow";
 import { Timeline } from "./comps/Timeline";
+import { SidePanelTabs } from "./comps/SidePanelTabs";
 
-import Editor from "./js/editor";
-import { TrackContext } from "~/contexts/TrackContext/TrackContext";
 import { TrackProvider } from "~/contexts/TrackContext/TrackProvider";
+import { EngineProvider } from "~/contexts/EngineProvider";
 
 export const PageEnigma = () => {
-  // const { setTopBarInner } = useContext(TopBarInnerContext) || {};
-
   const canvasRef = useRef<HTMLCanvasElement | null>(null);
-  const editorRef = useRef<Editor | null>(null);
+ 
   const [timelineHeight, setTimelineHeight] = useState(0);
   const timelineRef = useRef<HTMLDivElement | null>(null);
 
-  const editorCallback = useCallback(() => {
-    // handle editorCallback here
-  }, []);
-
-  useEffect(() => {
-    //componentDidMount
-
-    if (editorRef.current == null) {
-      editorRef.current = new Editor();
-      editorRef.current.initialize();
-    }
-  }, []);
-
-  const handleButtonSave = () => {
-    editorRef.current?.save();
-  };
-
-  const handleButtonCameraView = () => {
-    editorRef.current?.change_camera_view();
-  };
-
-  const handleButtonPlayBack = () => {
-    editorRef.current?.start_playback();
-  };
 
   const updateTimelineHeight = useCallback(() => {
     if (timelineRef.current) {
@@ -72,68 +47,58 @@ export const PageEnigma = () => {
   return (
     <div>
       <TopBarHelmet>
-        <Button icon={faWandSparkles}>Generate Movie</Button>
+        <Button icon={faSparkles}>Generate Movie</Button>
       </TopBarHelmet>
+      <EngineProvider>
+        <div style={{ height: "calc(100vh - 68px)" }}>
+          {/* Engine section/side panel */}
+          <div
+            id="CanvasUiWrapper"
+            className="flex"
+            // style={{ height: `calc(100% - ${timelineHeight}px)` }}
+            style={{ height: `calc(100% - 260px` }}
+          >
+            <div className="relative w-full overflow-hidden bg-gray-400">
+              <canvas
+                ref={canvasRef}
+                id="video-scene"
+                width="1280px"
+                height="720px"
+              />
 
-      <div style={{ height: "calc(100vh - 68px)" }}>
-        {/* Engine section/side panel */}
-        <div
-          className="flex"
-          style={{ height: `calc(100% - ${timelineHeight}px)` }}
-        >
-          <div className="relative w-full overflow-hidden bg-gray-400">
-            <canvas
-              ref={canvasRef}
-              id="video-scene"
-              width="1280px"
-              height="720px"
-            />
-
-            {/* Top controls */}
-            <div className="absolute left-0 top-0 w-full">
-              <div className="grid grid-cols-3 gap-4">
-                <div className="flex gap-2 pl-3 pt-3">
-                  <Button variant="secondary" onClick={handleButtonPlayBack}>
-                    Toggle Camera View
-                  </Button>
-                  <Button variant="secondary" onClick={handleButtonSave}>
-                    Save Scene
-                  </Button>
-                  <ButtonDialogue
-                    buttonProps={{
-                      variant: "secondary",
-                      label: "Help",
-                    }}
-                    title="Help"
-                  >
-                    <p>Do you need help?</p>
-                    <p>Ask Michael about this project</p>
-                    <p>Ask Miles about ThreeJS</p>
-                    <p>Ask Wil about React</p>
-                  </ButtonDialogue>
+              {/* Top controls */}
+              <div className="absolute left-0 top-0 w-full">
+                <div className="grid grid-cols-3 gap-4">
+                  <ControlsTopButtons />
+                  <Controls3D />
                 </div>
+              </div>
 
-                <Controls3D />
+              {/* Bottom controls */}
+              <div className="absolute bottom-0 left-0 w-full">
+                <PreviewWindow />
+                <ControlsVideo />
               </div>
             </div>
 
-            {/* Bottom controls */}
-            <div className="absolute bottom-0 left-0 w-full">
-              <ControlsVideo />
-            </div>
+            {/* Side panel */}
+            <SidePanel>
+              <SidePanelTabs />
+            </SidePanel>
           </div>
 
-          {/* Side panel */}
-          <SidePanel />
+          {/* Timeline */}
+          <div className="min-h-[220px]" ref={timelineRef}>
+            <TrackProvider>
+              <Timeline
+                // editorCurrent={editorRef.current}
+                // time={20}
+                // length={12}
+              />
+            </TrackProvider>
+          </div>
         </div>
-
-        {/* Timeline */}
-        <div className="min-h-[260px]" ref={timelineRef}>
-          <TrackProvider>
-            <Timeline editorCurrent={editorRef.current} length={12} />
-          </TrackProvider>
-        </div>
-      </div>
+      </EngineProvider>
     </div>
   );
 };

app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx

@@ -0,0 +1,80 @@
+import { 
+  useContext,
+  useState
+} from "react";
+import { Button, H4 } from "~/components";
+import { ButtonDialogue } from "~/modules/ButtonDialogue";
+import { EngineContext } from "~/contexts/EngineContext";
+
+
+export const ControlsTopButtons = () => {
+  const editorEngine = useContext(EngineContext);
+  const [sceneName, setSceneName] = useState<string>("");
+
+  const handleButtonSave = () => {
+    console.log(`SceneName is ${sceneName}`)
+    // editorEngine?.save();
+  };
+
+  const handleButtonCameraView = () => {
+    editorEngine?.change_camera_view();
+  };
+
+  const handleButtonPlayBack = () => {
+    editorEngine?.start_playback();
+  };
+  const handleButtonLoad = () => {
+    document.getElementById("load-upload")?.click();
+  };
+  const handleButtonRender = () => {
+    console.log("Saving GLB to server for reference")
+    editorEngine._upload_for_testing()
+    //editorEngine?.togglePlayback();
+  };
+  const handleButtonPlay = () => {};
+
+  return(
+    <div className="flex gap-2 pl-3 pt-3">
+      <Button variant="secondary" onClick={handleButtonPlayBack}>
+        Toggle Camera View
+      </Button>
+
+      <ButtonDialogue
+        buttonProps={{
+          variant: "secondary",
+          label: "Save Scene",
+        }}
+        title="Save Scene"
+        confirmButtonProps={{
+          label: "Save",
+          disabled:(sceneName === "" ? true : false),
+          onClick:(handleButtonSave)
+        }}
+      >
+        <H4 className="text-black">Please Enter a name for your scene</H4>
+        <input type="text" onChange={(e)=>{
+          setSceneName(e.target.value)
+        }}/>
+      </ButtonDialogue>
+
+      <ButtonDialogue
+        buttonProps={{
+          variant: "secondary",
+          label: "Help",
+        }}
+        title="Help"
+      >
+        <p>Do you need help?</p>
+        <p>Ask Michael about this project</p>
+        <p>Ask Miles about ThreeJS</p>
+        <p>Ask Wil about React</p>
+      </ButtonDialogue>
+
+      <div className="fixed flex gap-2" style={{ top: 124, left: 12}}>
+        <Button onClick={handleButtonLoad}>Load</Button>
+        <Button onClick={handleButtonRender}>Render</Button>
+        <Button onClick={handleButtonPlay}>Play</Button>
+      </div>
+    </div>
+  );
+};

app/pages/PageEnigma/comps/ControlsTopButtons/index.tsx

@@ -0,0 +1,2 @@
+export { ControlsTopButtons } from './ControlsTopButtons';
+

app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx

@@ -24,16 +24,14 @@ export const ControlsVideo = () => {
     console.log("Controls Video: Forward-Fast clicked");
   };
   return (
-    <div>
-      <div className="flex justify-center">
-        <div className="rounded-t-lg border-x border-t border-ui-panel-border bg-ui-controls p-2 text-white">
-          <div className="flex content-center gap-2">
-            <ButtonIcon icon={faBackwardFast} onClick={handleBackwardFast} />
-            <ButtonIcon icon={faBackwardStep} onClick={handleBackwardStep} />
-            <ButtonIcon icon={faPlay} onClick={handlePlay} />
-            <ButtonIcon icon={faForwardStep} onClick={handleForwardStep} />
-            <ButtonIcon icon={faForwardFast} onClick={handleForwardFast} />
-          </div>
+    <div className="flex justify-center">
+      <div className="rounded-t-lg border-x border-t border-ui-panel-border bg-ui-controls p-2 text-white">
+        <div className="flex content-center gap-2">
+          <ButtonIcon icon={faBackwardFast} onClick={handleBackwardFast} />
+          <ButtonIcon icon={faBackwardStep} onClick={handleBackwardStep} />
+          <ButtonIcon icon={faPlay} onClick={handlePlay} />
+          <ButtonIcon icon={faForwardStep} onClick={handleForwardStep} />
+          <ButtonIcon icon={faForwardFast} onClick={handleForwardFast} />
         </div>
       </div>
     </div>

app/pages/PageEnigma/comps/PreviewWindow/index.tsx

@@ -0,0 +1,30 @@
+import { useContext } from "react"
+import { EngineContext } from "~/contexts/EngineContext"
+
+export const PreviewWindow = ()=>{
+  const editorEngine = useContext(EngineContext);
+  //take data from egine context
+
+  return (
+    <div
+      id="preview-window"
+      className="absolute bottom-0 w-30 m-4"
+    >
+      <div className="relative">
+        <div
+          id="preview-window-label"
+          className="text-white bg-ui-panel pt-1 px-2 pb-3 -z-10 rounded-t-lg -mb-2 w-fit"
+        >
+          <p>Camera View</p>
+        </div>
+        <div className="rounded-lg border border-white box overflow-hidden">
+          <img
+            //shoot that datat from engine context to this image
+            className="aspect-video max-h-40" 
+            src="/resources/uiAssets/video_player_placeholder.gif"
+          />
+        </div>
+      </div>
+    </div>
+  )
+}

app/pages/PageEnigma/comps/SidePanelTabs/TabStyling.tsx

@@ -0,0 +1,53 @@
+
+import {
+  H4,
+  ItemPicker,
+  Textarea
+} from "~/components"
+
+
+export const TabStyling = ()=>{
+  const handlePickingStyle = (picked:string)=>{
+    console.log(`Picked style: ${picked}`);
+  }
+
+  return(
+    <>
+      <H4>Select Base Style</H4>
+      <div className="flex gap-2 my-2">
+        <ItemPicker
+          label="Anime"
+          onClick={()=>handlePickingStyle("anime")}
+          src="/resources/avatars/0.webp"
+        />
+        <ItemPicker
+          label="Pixel"
+          onClick={()=>handlePickingStyle("pixel")}
+          src="/resources/avatars/1.webp"
+        />
+        <ItemPicker
+          label="Pixar"
+          onClick={()=>handlePickingStyle("pixar")}
+          src="/resources/avatars/2.webp"
+        />
+        <ItemPicker
+          label="Stylized"
+          onClick={()=>handlePickingStyle("stylized")}
+          src="/resources/avatars/3.webp"
+        />
+      </div>
+      <H4>Enter a Prompt</H4>
+      <div className="flex gap-2 my-2">
+        <Textarea className="w-full" />
+      </div>
+      <H4>Negative Prompt</H4>
+      <div className="flex gap-2 my-2">
+        <Textarea
+          className="w-full"
+          placeholder="Type here to filter out the things you don't want in the scene
+        "/>
+      </div>
+    </>
+  );
+
+}

app/pages/PageEnigma/comps/SidePanelTabs/index.tsx

@@ -0,0 +1,28 @@
+import { Tabs } from "~/modules/Tabs";
+import { TabStyling } from "./TabStyling";
+
+export const SidePanelTabs = ()=>{
+  return(
+    <Tabs
+      tabs={[
+        {
+          header: "Animation",
+          children: <p>Animation Tab</p>,
+        },
+        {
+          header: "Camera",
+          children: <p>Camera Tab</p>,
+        },
+        {
+          header: "Audio",
+          children: <p>Audio Tab</p>,
+        },
+        {
+          header: "Styling",
+          children: <TabStyling />,
+        },
+      ]}
+    />
+  );
+
+}

app/pages/PageEnigma/comps/Timeline/Timeline.tsx

@@ -1,9 +1,20 @@
-import { LowerPanel } from "~/modules/LowerPanel";
-import Editor from "../../js/editor";
-import { Character } from "./Character";
-import { Fragment, useContext } from "react";
-import { TrackContext } from "~/contexts/TrackContext/TrackContext";
+import { 
+  Fragment,
+  // useCallback,
+  useContext
+} from "react";
+
 import { Button } from "~/components";
+
+// import Editor from "../../js/editor";
+import { EngineContext } from "~/contexts/EngineContext";
+import { TrackContext } from "~/contexts/TrackContext/TrackContext";
+
+import { LowerPanel } from "~/modules/LowerPanel";
+import { Character } from "./Character";
+
+
+
 import { Camera } from "./Camera";
 import { Audio } from "./Audio";
 import { Objects } from "./Objects";
@@ -12,32 +23,24 @@ import { faSortDown } from "@fortawesome/pro-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 
 export const Timeline = ({
-  editorCurrent,
+  // editorCurrent,
+  // time,
 }: {
-  editorCurrent: Editor | null;
+  // editorCurrent: Editor | null;
+  // time: number;
 }) => {
+  const editorEngine = useContext(EngineContext);
+  // editorEngine replaced the engineRef.current passed as a prop
+  // const { characters, updateCharacters } = useContext(TrackContext);
   const { characters, objects, scale, length } = useContext(TrackContext);
   const { onPointerDown, time } = useMouseEventsAnimation();
 
-  const handleButtonLoad = () => {
-    document.getElementById("load-upload")?.click();
-  };
-  const handleButtonRender = () => {
-    editorCurrent?.togglePlayback();
-  };
-  const handleButtonPlay = () => {};
-
   const sectionWidth = 60 * 4 * scale;
   const fullHeight =
     characters.length * 200 + objects.objects.length * 80 + 248 + 96;
 
   return (
     <>
-      <div className="fixed" style={{ top: 72, left: "calc(100% - 600px)" }}>
-        <Button onClick={handleButtonLoad}>Load</Button>
-        <Button onClick={handleButtonRender}>Render</Button>
-        <Button onClick={handleButtonPlay}>Play</Button>
-      </div>
       <LowerPanel>
         <div className="mt-4 flex h-3 text-sm text-white">
           {Array(length)

app/pages/PageEnigma/datastructures/clips/animation_track_clip.ts

@@ -66,35 +66,39 @@ export class AnimationTrackClip implements AnimationTrackClip {
 
   _create_mixer(object: THREE.Object3D) {
     this.mixer = new THREE.AnimationMixer(object);
+    return this.mixer;
   }
 
   async _get_clip() {
     if (this.animation_clip == null) {
-      this.animation_clip = await this._load_animation(this.clip_name);
+      this.animation_clip = await this._load_animation(this.media_id);
     }
     return this.animation_clip;
   }
 
   async play(object: THREE.Object3D) {
-    if (this.mixer == null) { this._create_mixer(object) }
-
-    let anim_clip = await this._get_clip();
-    this.clip_action = this.mixer?.clipAction(anim_clip);
-    if (this.clip_action) {
-      if (this.clip_action?.isRunning() == false) {
-        this.clip_action.play();
+    if (this.mixer == null) {
+      this.mixer = this._create_mixer(object);
+      let anim_clip = await this._get_clip();
+      this.clip_action = this.mixer?.clipAction(anim_clip);
+      if (this.clip_action) {
+        if (this.clip_action?.isRunning() == false) {
+          this.clip_action.play();
+        }
       }
+      console.log("Play")
     }
-
   }
 
   step(deltatime: number) {
     if (this.mixer == null) { return; }
     this.mixer?.update(deltatime);
+    console.log("Update")
   }
 
   stop() {
     this.mixer?.stopAllAction();
+    console.log("stop.")
   }
 
   toJSON(): string {

app/pages/PageEnigma/datastructures/clips/clip_offset.ts

@@ -1,17 +1,9 @@
-export interface ClipUI {
-  version: number
-  type: "transform" | "audio" | "animation" | "lipsync"
-  media_id: string
-  object_uuid: string
-  start_offset: number // in frames so 60 frames per second on the timeline.
-  ending_offset: number // ending offset
-}
-
 // Clip offsets represent the state of the clip on the timeline as well as what type of clip it is.
 // it is created from a media id.
-export class ClipUI implements ClipUI {
+export class ClipUI {
   version: number
   type: "transform" | "audio" | "animation" | "lipsync"
+  name: string
   media_id: string
   object_uuid: string
   start_offset: number // in frames
@@ -20,12 +12,14 @@ export class ClipUI implements ClipUI {
   constructor(
     version: number,
     type: "transform" | "audio" | "animation" | "lipsync",
+    name: string,
     media_id: string,
     object_uuid: string,
     start_offset: number,
     ending_offset: number,
   ) {
     this.version = version
+    this.name = name
     this.type = type
     this.object_uuid = object_uuid
     this.media_id = media_id
@@ -36,6 +30,7 @@ export class ClipUI implements ClipUI {
   toJSON(): string {
     return JSON.stringify({
       version: this.version,
+      name: this.name,
       type: this.type,
       object_uuid: this.object_uuid,
       media_id: this.media_id,

app/pages/PageEnigma/datastructures/clips/lipsync_track_clip.ts

@@ -1,99 +1,134 @@
 import { LipSync } from "../../js/lipsync";
 import * as THREE from 'three';
 interface AudioData {
-    audioContext: AudioContext;
-    audioBuffer: AudioBuffer;
+  audioContext: AudioContext;
+  audioBuffer: AudioBuffer;
+}
+
+class AudioData implements AudioData {
+  audioContext: AudioContext;
+  audioBuffer: AudioBuffer;
+  source: AudioBufferSourceNode | undefined
+
+  constructor(audioContext: AudioContext, audioBuffer: AudioBuffer) {
+    this.audioContext = audioContext;
+    this.audioBuffer = audioBuffer;
   }
-  
-  class AudioData implements AudioData{
-    audioContext: AudioContext;
-    audioBuffer: AudioBuffer;
-    source: AudioBufferSourceNode | undefined
-  
-    constructor (audioContext: AudioContext, audioBuffer: AudioBuffer) {
-      this.audioContext = audioContext;
-      this.audioBuffer = audioBuffer;
-    }
+}
+
+export interface LipSyncTrackClip {
+  version: number;
+  media_id: string;
+  type: "lipsync";
+  volume: number;
+}
+
+class BlendShapeHelper {
+  ah: number;
+  ee: number;
+  oh: number;
+  constructor(ah: number, ee: number, oh: number) {
+    this.ah = ah;
+    this.ee = ee;
+    this.oh = oh;
   }
-  
-  export interface LipSyncTrackClip {
-    version: number;
-    media_id: string;
-    type: "lipsync";
-    volume: number;
+}
+
+export class LipSyncTrackClip implements LipSyncTrackClip {
+  version: number;
+  media_id: string;
+  type: "lipsync" = "lipsync";
+  volume: number;
+  audio_data: AudioData | undefined;
+  lipsync: LipSync;
+  blendshape_helper: BlendShapeHelper;
+
+  constructor(version: number, media_id: string, volume: number) {
+    this.version = version;
+    this.media_id = media_id;
+    this.type = "lipsync";
+    this.volume = volume;
+    this.download_audio().then(data => {
+      this.audio_data = data;
+    });
+    this.blendshape_helper = new BlendShapeHelper(0, 0, 0);
+    // we might need 3 of these one for each character ...
+    this.lipsync = new LipSync();
   }
-  
-  export class LipSyncTrackClip implements LipSyncTrackClip {
-    version: number;
-    media_id: string;
-    type: "lipsync" = "lipsync";
-    volume: number;
-    audio_data: AudioData | undefined;
-    lipsync: LipSync;
-  
-    constructor(version: number, media_id: string, volume: number) {
-      this.version = version;
-      this.media_id = media_id;
-      this.type = "lipsync";
-      this.volume = volume;
-      this.download_audio().then(data => {
-        this.audio_data = data;
+
+  // lip sync will be generated through TTS 
+  async get_media_url() {
+    //This is for prod when we have the proper info on the url.
+    let api_base_url = "https://api.fakeyou.com";
+    let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
+    let responce = await fetch(url);
+    let json = await JSON.parse(await responce.text());
+    let bucketPath = json["media_file"]["public_bucket_path"];
+    let media_base_url = "https://storage.googleapis.com/vocodes-public"
+    let media_url = `${media_base_url}${bucketPath}`
+    return media_url;
+  }
+
+  async download_audio() {
+    let url = await this.get_media_url();
+    const audioContext = new AudioContext();
+    const response = await fetch(url);
+    const arrayBuffer = await response.arrayBuffer();
+    const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+    return new AudioData(audioContext, audioBuffer);
+  }
+
+  async _detect_face(object: THREE.Object3D): Promise<THREE.Mesh> {
+    return new Promise((resolve) => {
+      object.traverse((c: THREE.Object3D) => {
+        if (c instanceof THREE.Mesh) {
+          if (c.morphTargetInfluences && c.morphTargetDictionary) {
+            const blendShapeIndexE = c.morphTargetDictionary["E"];
+            const blendShapeIndexO = c.morphTargetDictionary["O"];
+            const blendShapeIndexA = c.morphTargetDictionary["aa"];
+            if (blendShapeIndexE != null) {
+              this.blendshape_helper = new BlendShapeHelper(blendShapeIndexA, blendShapeIndexE, blendShapeIndexO);
+              resolve(c);
+              return;
+            }
+          }
+        }
       });
-
-      // we might need 3 of these one for each character ...
-      this.lipsync = new LipSync()
-    }
-    
-    // lip sync will be generated through TTS 
-    async get_media_url() {
-      //This is for prod when we have the proper info on the url.
-      let api_base_url = "https://api.fakeyou.com";
-      let url = `${api_base_url}/v1/media_files/file/${this.media_id}`
-      let responce = await fetch(url);
-      let json = await JSON.parse(await responce.text());
-      let bucketPath = json["media_file"]["public_bucket_path"];
-      let media_base_url = "https://storage.googleapis.com/vocodes-public"
-      let media_url = `${media_base_url}${bucketPath}`
-      return media_url;
-    }
-  
-    async download_audio() {
-      let url = await this.get_media_url();
-      const audioContext = new AudioContext();
-      const response = await fetch(url);
-      const arrayBuffer = await response.arrayBuffer();
-      const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
-      return new AudioData(audioContext, audioBuffer);
-    }
-
-    play(face: THREE.Object3D) {
-      if (this.audio_data?.audioBuffer == null) { this.download_audio(); }
-      this.lipsync = new LipSync()
-      this.lipsync.face = face;
-      this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
-    }
-
-    stop() {
-      if(this.lipsync == null) { return; }
-      this.lipsync.destroy();
-    }
-
-    step() {
-      if(this.lipsync == null) { return; }
-      this.lipsync.update();
-    }
-
-    reset() {
-      this.lipsync = new LipSync()
-    }
-  
-    toJSON(): string {
-      return JSON.stringify({
-        version: this.version,
-        media_id: this.media_id,
-        type: this.type,
-        volume: this.volume,
-      });
-    }
+    });
   }
-  
+
+
+  async play(object: THREE.Object3D) {
+  if (this.audio_data?.audioBuffer == null) { await this.download_audio(); }
+  if (this.lipsync.face == null) {
+    this.lipsync = new LipSync(await this._detect_face(object));
+    this.lipsync.startLipSyncFromAudioBuffer(this.audio_data?.audioBuffer);
+  }
+}
+
+stop() {
+  if (this.lipsync == null) { return; }
+  this.lipsync.destroy();
+}
+
+step() {
+  if (this.lipsync == null) { return; }
+  const positions = this.lipsync.update();
+  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = positions["ee"];
+  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = positions["ah"];
+  this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = positions["oh"];
+}
+
+reset() {
+  this.lipsync = new LipSync();
+}
+
+toJSON(): string {
+  return JSON.stringify({
+    version: this.version,
+    media_id: this.media_id,
+    type: this.type,
+    volume: this.volume,
+  });
+}
+}

app/pages/PageEnigma/datastructures/clips/transform_track_clip.ts

@@ -68,7 +68,6 @@ export class TransformTrackClip implements TransformTrackClip {
       object.position.copy(first_pos);
       object.rotation.set(first_rot.x, first_rot.y, first_rot.z);
       object.scale.copy(first_scl);
-      
       this.step_frame = 0;
     }
   }

app/pages/PageEnigma/js/api_manager.ts

@@ -1,54 +1,67 @@
-import { v4 as uuidv4 } from 'uuid';
+import { v4 as uuidv4 } from 'uuid'
+import { Scene } from '../datastructures/scene/scene_object'
+import * as THREE from 'three'
+import { GLTFExporter } from 'three/addons/exporters/GLTFExporter.js'
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
+import { TimelineCurrentReactState } from './timeline'
 
-import { Scene } from '../datastructures/scene/scene_object';
-
-class MediaUploadManager {
+// Basically the data will exist on the database 
+class APIManager {
   baseUrl:String
 
   constructor() {
-    this.baseUrl = "https://api.fakeyou.com";
+    this.baseUrl = "https://api.fakeyou.com"
     //this.baseUrl = "http://localhost:12345"
   }
   
-  async saveScene(scene:Scene):Promise<string> {
+
+  // Function to save or update the scene
+  async saveSceneState(scene:THREE.Scene,
+                      timeLine:TimelineCurrentReactState,
+                      scene_id:string | null =null):Promise<string> { 
+    if (scene_id == null) {
+      // create new scene.
+      let file = await this.gltfExport(scene)
+      let response = await this.uploadGLB(file)
+      console.log(response)
+    } else {
+      // update scene or overwrite glb scene.
+    }
     return ""
   }
 
-  // two core functions to upload things to the server and get id's
-  async uploadMedia(blob:any, fileName:string):Promise<string> {
-    const url = `${this.baseUrl}/v1/media_uploads/upload`;
-    let uuid = uuidv4();
-    const formData = new FormData();
-    formData.append('uuid_idempotency_token', uuid);
-    formData.append('file', blob, fileName);
-    formData.append('source', 'file');
-    formData.append('type', 'video');
-    formData.append('source', 'file');
-    const response = await fetch(url, {
-      method: 'POST',
-      credentials: "include",
-      headers: {
-        "Accept": "application/json",
-      },
-      body: formData,
-    });
-
-    if (!response.ok) {
-      throw new Error('Failed to Send Data');
-    }
-
-    return response.json(); // or handle the response as appropriate
+  async _saveSceneAndTimelineToJSONSpec() {
+    
   }
 
-  async uploadGLB(file:File):Promise<string> {
-    const url = `${this.baseUrl}/v1/media_files/upload/engine_asset`;
-    let uuid = uuidv4();
-    const formData = new FormData();
-    formData.append('uuid_idempotency_token', uuid);
-    formData.append('file', file);
-    formData.append('source','file');
-    formData.append('media_file_subtype', 'scene_import');
-    formData.append('media_file_class', 'scene');
+  async loadScene(scene_token:[]):Promise<string> {
+    
+    return ""
+  }
+
+  async batchMedia(media_tokens:[]):Promise<string> {
+
+    return ""
+  }
+
+  async gltfExport(scene:THREE.Scene) {
+      let gltfExporter = new GLTFExporter()    
+      let uuid = uuidv4()  
+      // save the scene to keep ids positions etc
+      let result = await gltfExporter.parseAsync(scene)
+      const file = new File([JSON.stringify(result)], `${uuid}.glb`, {type: 'application/json'})
+      return file
+  }
+  // two core functions to upload things to the server and get id's
+  async uploadMedia(blob:any, fileName:string):Promise<string> {
+    const url = `${this.baseUrl}/v1/media_uploads/upload`
+    let uuid = uuidv4()
+    const formData = new FormData()
+    formData.append('uuid_idempotency_token', uuid)
+    formData.append('file', blob, fileName)
+    formData.append('source', 'file')
+    formData.append('type', 'video')
+    formData.append('source', 'file')
     const response = await fetch(url, {
       method: 'POST',
       credentials: "include",
@@ -56,13 +69,39 @@ class MediaUploadManager {
         "Accept": "application/json",
       },
       body: formData,
-    });
+    })
 
     if (!response.ok) {
-      throw new Error('Failed to Send Data');
+      throw new Error('Failed to Send Data')
     }
-    return response.json(); // or handle the response as appropriate
+
+    return response.json() // or handle the response as appropriate
+  }
+
+  // to save to a file then upload the scene.
+  async uploadGLB(file:File):Promise<string> {
+    const url = `${this.baseUrl}/v1/media_files/upload/engine_asset`
+    let uuid = uuidv4()
+    const formData = new FormData()
+    formData.append('uuid_idempotency_token', uuid)
+    formData.append('file', file)
+    formData.append('source','file')
+    formData.append('media_file_subtype', 'scene_import')
+    formData.append('media_file_class', 'scene')
+    const response = await fetch(url, {
+      method: 'POST',
+      credentials: "include",
+      headers: {
+        "Accept": "application/json",
+      },
+      body: formData,
+    })
+
+    if (!response.ok) {
+      throw new Error('Failed to Send Data')
+    }
+    return response.json() // or handle the response as appropriate
   }
 }
 
-export default MediaUploadManager;
+export default APIManager

app/pages/PageEnigma/js/editor.js

@@ -3,7 +3,7 @@ import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
 import { TransformControls } from 'three/addons/controls/TransformControls.js';
 import Scene from './scene.js';
 import SaveManager from './serialization.js';
-import MediaUploadManager from './api_manager.ts';
+import APIManager from './api_manager.ts';
 import { EffectComposer } from 'three/addons/postprocessing/EffectComposer.js';
 import { RenderPass } from 'three/addons/postprocessing/RenderPass.js';
 import { OutlinePass } from 'three/addons/postprocessing/OutlinePass.js';
@@ -23,6 +23,7 @@ import { LipSync } from './lipsync.js';
 import { LipSyncEngine } from "./lip_sync_engine.ts";
 import { AnimationEngine} from "./animation_engine.ts";
 import { faL } from '@fortawesome/pro-solid-svg-icons';
+
 if (typeof window !== 'undefined') {
     import('ccapture.js').then(module => {
         const CCapture = module.CCapture;
@@ -79,7 +80,7 @@ class Editor {
         this.transform_interaction;
         this.rendering = false;
         // API.
-        this.api_manager = new MediaUploadManager();
+        this.api_manager = new APIManager();
         // Debug & Movement.
         this.stats = null;
         this.orbit = null;
@@ -155,21 +156,36 @@ class Editor {
 
         this.timeline.scene = this.activeScene;
         
-
-     
         this._test_demo()
     }
 
-    async _test_demo() {
-        // Test code here
-        // this.test_box_uuid = this.activeScene.instantiate("Box");
-        // let object = this.transform_engine.loadObject(this.test_box_uuid);
-        // this.timeline.addPlayableClip(new ClipUI(1.0, "transform", object.object_uuid, object.media_id, 50, 300));
-        // this.timeline.addPlayableClip(new ClipUI(1.0, "audio", "m_f7jnwt3d1ddchatdk5vaqt0n4mb1hg", null, 50, 50));
+    // uploading some objects for testing to get their media ids from my account.
+    async _upload_for_testing() {
+        // have to upload as a file first
 
+        // This is the default scene.
+        // "m_189p8hj0eyypbg74kkhcpehwpjhnkz" scene with the fox.
+        let result = await this.api_manager.saveSceneState(this.activeScene.scene)
+        console.log(result)
+    }
+
+    async _test_demo() {
+        // note the database from the server is the source of truth for all the data.
+        // Test code here
         let object = await this.activeScene.load_glb("./resources/models/fox/fox.glb")
+
+        // load object into the engine for lip syncing
         this.lipsync_engine.load_object(object.uuid, "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72");
-        this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", object.object_uuid, object.media_id, 0, 100));
+
+        // create the clip with the same id for a reference to the media
+        this.timeline.addPlayableClip(new ClipUI(1.0, "lipsync", "clip1", "m_f1jxx4zwy4da2zn0cvdqhha7kqkj72", object.uuid, 150, 400));
+        
+        // media id for this is up in the air but when a path is created you should be able to store and delete it
+        this.timeline.addPlayableClip(new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, 0, 150));
+
+        // media id for this as well it can be downloaded
+        this.timeline.addPlayableClip(new ClipUI(1.0, "animation", "clip3", "/resources/models/fox/fox_idle.glb", object.uuid, 0, 400));
+        this.animation_engine.load_object(object.uuid, "/resources/models/fox/fox_idle.glb", "clip3");
     }
 
     // Configure post processing.
@@ -255,6 +271,7 @@ class Editor {
         
         if(this.selected == null) {return;}
         this.transform_engine.addFrame(this.selected)
+        console.log("Frame taken.")
     }
 
     change_camera_view() {
@@ -272,26 +289,27 @@ class Editor {
         // Updates debug stats.
         if (this.stats != null) { this.stats.update(); }
 
+        let delta_time = this.clock.getDelta()
+
         // All calls that are not super important like timeline go here.
-        this.activeScene.update(this.clock.getDelta());
+        this.activeScene.update(delta_time);
         //this.orbit.update(0.1);
 
         //console.log(this.transform_engine.clips[this.test_box_uuid]);
 
-        this.timeline.update(this.clock.getDelta());
+        this.timeline.update(delta_time);
 
         this.render_scene();
         if (this.capturer != null) { this.capturer.capture(this.renderer.domElement); } // Record scene.
-
     }
 
     start_playback() {
         this.timeline.isPlaying = true;
     }
 
-    _save_to_cloud(blob) {
-        this.api_manager.uploadGLB(blob, "test.glb");
-    }
+    // _save_to_cloud(blob) {
+    //     this.api_manager.uploadGLB(blob, "test.glb");
+    // }
 
     change_mode(type) {
         this.control.mode = type;

app/pages/PageEnigma/js/lipsync.js

@@ -18,7 +18,7 @@ for (let m = 0; m < BoundingFrequencyFem.length; m++) {
 }
 
 export class LipSync {
-  constructor(face=null) {
+  constructor(face) {
 
     this.face = face;
 
@@ -90,19 +90,15 @@ export class LipSync {
   update() {
     if (this.meter) {
       const { volume } = this.meter;
+      let { ah, oh, ee } = this.update2();
       if (volume < 0.01) {
-        // get the shape keys from the face avatar
-
-        this.face.morphTargetInfluences[this.ee] = 0;
-        this.face.morphTargetInfluences[this.ah] = 0;
-        this.face.morphTargetInfluences[this.oh] = 0;
-      } else {
-        const { ah, oh, ee } = this.update2();
-        this.face.morphTargetInfluences[this.ee] = ee;
-        this.face.morphTargetInfluences[this.ah] = ah;
-        this.face.morphTargetInfluences[this.oh] = oh;
+        ah = 0;
+        oh = 0;
+        ee = 0;
       }
+      return { "ah": ah, "oh": oh, "ee": ee };
     }
+    return { "ah": 0, "oh": 0, "ee": 0 };
   }
 
   update2() {

app/pages/PageEnigma/js/timeline.ts

@@ -117,12 +117,15 @@ export class TimeLine {
         for (const element of this.timelineItems) {
             if (element.type == "transform") {
                 let object = this.scene.get_object_by_uuid(element.object_uuid);
-                if (object) { this.transformEngine.clips[element.object_uuid].reset(object); }
+                if (object && this.transformEngine.clips[element.object_uuid]) { this.transformEngine.clips[element.object_uuid].reset(object); }
             }
             else if (element.type == "audio") {
                 this.audioEngine.loadClip(element.media_id);
             }
             else if (element.type == "animation") {
+            }
+            else if (element.type == "lipsync") {
+                
             }
             else {
                 this.stop()
@@ -140,10 +143,8 @@ export class TimeLine {
         }
 
         this.scrubberPosition += 1;
-
         //2. allow stopping.
         //3. smallest unit is a frame and it is set by the scene and is in fps, our videos will be 60fps but we can reprocess them using the pipeline.
-
         for (const element of this.timelineItems) {
             if (element.start_offset <= this.scrubberPosition && this.scrubberPosition <= element.ending_offset) {
                 // run async
@@ -151,7 +152,7 @@ export class TimeLine {
                 // remove the element from the list
                 let object = this.scene.get_object_by_uuid(element.object_uuid)
                 if (element.type == "transform") {
-                    if (object) {
+                    if (object && this.transformEngine.clips[element.object_uuid]) {
                         this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset)
                         this.transformEngine.clips[element.object_uuid].step(object)
                     }
@@ -161,15 +162,15 @@ export class TimeLine {
                     this.audioEngine.playClip(element.media_id)
                 }
                 else if (element.type == "lipsync") {
-                    let face_object = this.scene.get_object_by_uuid(element.object_uuid);
-                    if (face_object) { 
-                        this.lipSyncEngine.clips[element.object_uuid].play(face_object);
+                    if(this.scrubberPosition+1 >= element.ending_offset){
+                        this.lipSyncEngine.clips[element.object_uuid].stop();
+                    }
+                    else if (object) {
+                        await this.lipSyncEngine.clips[element.object_uuid].play(object);
                         this.lipSyncEngine.clips[element.object_uuid].step();
                     }
                 }
-
                 else if (element.type == "animation") {
-                    let object = this.scene.get_object_by_uuid(element.object_uuid);
                     if (object) { 
                         await this.animationEngine.clips[object.uuid].play(object); 
                         this.animationEngine.clips[object.uuid].step(deltatime);

app/pages/PageEnigma/notes/note_batch_media_endpoint.json

@@ -0,0 +1,113 @@
+https://api.fakeyou.com/v1/media_files/batch?tokens=m_jh74zz1nfj2jxrkw7n00yamt32zrhb&tokens=m_0rqwx4kbnn2gz5s37apzftkw25vkzb&tokens=m_je04hbk7bqjdq9cnegp1j10mf4dzdb
+{
+    "success": true,
+    "media_files": [
+        {
+            "token": "m_0rqwx4kbnn2gz5s37apzftkw25vkzb",
+            "media_type": "video",
+            "media_class": "unknown",
+            "maybe_media_subtype": null,
+            "maybe_engine_extension": null,
+            "maybe_batch_token": null,
+            "public_bucket_path": "/media/c/0/n/g/p/c0ngp9c570w66q3vhs0ygmwpasf2sfk1/videoc0ngp9c570w66q3vhs0ygmwpasf2sfk1mp4",
+            "maybe_model_weight_info": null,
+            "maybe_creator_user": {
+                "user_token": "U:937C1VN0S5G8H",
+                "username": "echelon",
+                "display_name": "echelon",
+                "gravatar_hash": "37e7e18f19d8f654a1ace405c846a9de",
+                "default_avatar": {
+                    "image_index": 10,
+                    "color_index": 3
+                }
+            },
+            "creator_set_visibility": "private",
+            "maybe_text_transcript": null,
+            "maybe_prompt_token": "prompt_3sf5p1n7g3sna6pc44nqgpvyd",
+            "maybe_original_filename": null,
+            "is_emulated_media_file": false,
+            "stats": {
+                "positive_rating_count": 0,
+                "bookmark_count": 0
+            },
+            "created_at": "2024-03-25T02:25:18Z",
+            "updated_at": "2024-03-25T02:25:18Z"
+        },
+        {
+            "token": "m_je04hbk7bqjdq9cnegp1j10mf4dzdb",
+            "media_type": "video",
+            "media_class": "unknown",
+            "maybe_media_subtype": null,
+            "maybe_engine_extension": null,
+            "maybe_batch_token": null,
+            "public_bucket_path": "/media/d/a/3/b/6/da3b640rh3zjj1easv96vc4d7v4jq8c1/da3b640rh3zjj1easv96vc4d7v4jq8c1.mp4",
+            "maybe_model_weight_info": null,
+            "maybe_creator_user": {
+                "user_token": "U:937C1VN0S5G8H",
+                "username": "echelon",
+                "display_name": "echelon",
+                "gravatar_hash": "37e7e18f19d8f654a1ace405c846a9de",
+                "default_avatar": {
+                    "image_index": 10,
+                    "color_index": 3
+                }
+            },
+            "creator_set_visibility": "public",
+            "maybe_text_transcript": null,
+            "maybe_prompt_token": null,
+            "maybe_original_filename": null,
+            "is_emulated_media_file": false,
+            "stats": {
+                "positive_rating_count": 0,
+                "bookmark_count": 0
+            },
+            "created_at": "2024-03-25T02:04:22Z",
+            "updated_at": "2024-03-25T02:04:22Z"
+        },
+        {
+            "token": "m_jh74zz1nfj2jxrkw7n00yamt32zrhb",
+            "media_type": "video",
+            "media_class": "unknown",
+            "maybe_media_subtype": null,
+            "maybe_engine_extension": null,
+            "maybe_batch_token": null,
+            "public_bucket_path": "/media/a/c/s/z/e/acszeddd464edypg9hcp5cga0482sjfv/videoacszeddd464edypg9hcp5cga0482sjfvmp4",
+            "maybe_model_weight_info": null,
+            "maybe_creator_user": {
+                "user_token": "U:937C1VN0S5G8H",
+                "username": "echelon",
+                "display_name": "echelon",
+                "gravatar_hash": "37e7e18f19d8f654a1ace405c846a9de",
+                "default_avatar": {
+                    "image_index": 10,
+                    "color_index": 3
+                }
+            },
+            "creator_set_visibility": "private",
+            "maybe_text_transcript": null,
+            "maybe_prompt_token": "prompt_dc8x82w2gqsvbpmp5ksxypssj",
+            "maybe_original_filename": null,
+            "is_emulated_media_file": false,
+            "stats": {
+                "positive_rating_count": 0,
+                "bookmark_count": 0
+            },
+            "created_at": "2024-03-25T02:12:46Z",
+            "updated_at": "2024-03-25T02:12:46Z"
+        }
+    ]
+}
+
+// lip sync will be generated through TTS 
+async get_media_url() {
+    //This is for prod when we have the proper info on the url.
+    let api_base_url = "https://api.fakeyou.com/";
+    let url = ${api_base_url}/v1/media_files/file/${this.media_id}
+    let responce = await fetch(url);
+    let json = await JSON.parse(await responce.text());
+    let bucketPath = json["media_file"]["public_bucket_path"];
+    let media_base_url = "https://storage.googleapis.com/vocodes-public"
+    let media_url = ${media_base_url}${bucketPath}
+    return media_url;
+  }
+https://storage.googleapis.com/vocodes-public

tailwind.config.ts

@@ -112,6 +112,7 @@ export default {
       "ui-divider": "#D3D3D3",
 
       gray: colors.gray,
+      black: colors.black,
       white: colors.white,
     },
   },

test/test_bench.js

@@ -1,205 +1,38 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
-    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+
+class Network {
+    constructor() { }
+
+    async sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
     }
-};
-var Clip = /** @class */ (function () {
-    function Clip(name, startOffset, lengthOfClip) {
-        this.startOffset = startOffset;
-        this.lengthOfClip = lengthOfClip;
-        this.name = name;
-        this.timerID = null;
-        this.scrubberEndingPosition = 100;
+    async task1() {
+        console.log("World.")
+        let val = true
+        while (val) {
+            await this.sleep(1000); 
+            console.log("Task1 doing work.")
+        }
+        //throw Error("Ded World.")
     }
-    Clip.prototype.play = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var _this = this;
-            return __generator(this, function (_a) {
-                console.log("Playing ".concat(this.name));
-                this.timerID = setInterval(function () { return __awaiter(_this, void 0, void 0, function () {
-                    return __generator(this, function (_a) {
-                        console.log("Running async ".concat(this.name));
-                        this.stop();
-                        return [2 /*return*/];
-                    });
-                }); }, this.lengthOfClip);
-                return [2 /*return*/];
-            });
-        });
-    };
-    Clip.prototype.stop = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                clearInterval(this.timerID);
-                console.log("Stopping ".concat(this.name));
-                return [2 /*return*/];
-            });
-        });
-    };
-    return Clip;
-}());
-var API = /** @class */ (function () {
-    function API() {
-        this.timelineItems = [];
-        this.timerID = null;
-        this.timeLineLimit = 1000 * 10;
+
+    async task2() {
+        console.log("Hello.")
+        let val = true 
+        while (val) {
+            await this.sleep(1000); 
+            console.log("Task2 doing work.")
+        }
+        //throw Error("Ded Hello.")
     }
-    API.prototype.addPlayableClip = function (clip) {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                this.timelineItems.push(clip);
-                return [2 /*return*/];
-            });
-        });
-    };
-    API.prototype.start = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var start, updateInterval, timerID;
-            var _this = this;
-            return __generator(this, function (_a) {
-                start = 0;
-                updateInterval = 100;
-                timerID = setInterval(function () { return __awaiter(_this, void 0, void 0, function () {
-                    var _loop_1, this_1, _i, _a, element;
-                    return __generator(this, function (_b) {
-                        console.log("Current Time:".concat(start));
-                        _loop_1 = function (element) {
-                            if (element.startOffset >= start) {
-                                element.play();
-                                // remove the element
-                                this_1.timelineItems = this_1.timelineItems.filter(function (item) { return item !== element; });
-                            }
-                        };
-                        this_1 = this;
-                        // get last updated time when stopped
-                        for (_i = 0, _a = this.timelineItems; _i < _a.length; _i++) {
-                            element = _a[_i];
-                            _loop_1(element);
-                        }
-                        if (start == this.timeLineLimit || this.timelineItems.length == 0) {
-                            this.stop();
-                        }
-                        start += updateInterval;
-                        return [2 /*return*/];
-                    });
-                }); }, updateInterval);
-                this.timerID = timerID;
-                return [2 /*return*/];
-            });
-        });
-    };
-    API.prototype.stop = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                console.log("Stopping Timeline");
-                clearInterval(this.timerID);
-                return [2 /*return*/];
-            });
-        });
-    };
-    return API;
-}());
-// How much timeline precision we have
-var percision = 100;
-// Visual verification tests.
-function CheckIfBasicClipWorks() {
-    var api = new API();
-    api.addPlayableClip(new Clip("clip1", 0, 1000));
-    api.start();
 }
-function CheckIfBasicClipWorks3SecondsAfterTimelineStops() {
-    var api = new API();
-    api.addPlayableClip(new Clip("clip1", 0, 10000));
-    api.start();
+
+const network = new Network() 
+
+async function main() {
+    network.task1()
+    network.task2()
 }
-function CheckIfTwoClipsAtTheSameTimeWorks() {
-    var api = new API();
-    api.addPlayableClip(new Clip("clip1", 0, 1000));
-    api.start();
-}
-function CheckIfTwoClipsOneAfterAnotherWorks() {
-    var api = new API();
-    api.addPlayableClip(new Clip("clip1", 0, 1000));
-    api.addPlayableClip(new Clip("clip2", 0, 2000));
-    api.start();
-}
-function CheckIfTimeLineStopBeforeClipPlays() {
-    var _this = this;
-    var api = new API();
-    api.addPlayableClip(new Clip("clip3", 0, 1000));
-    api.start();
-    setInterval(function () { return __awaiter(_this, void 0, void 0, function () {
-        return __generator(this, function (_a) {
-            api.stop();
-            return [2 /*return*/];
-        });
-    }); }, 1100);
-    console.log("Stopped");
-}
-function CheckIfTimeLineStartAfterClipPlays() {
-    var _this = this;
-    var api = new API();
-    api.addPlayableClip(new Clip("clip3", 0, 1000));
-    api.start();
-    setInterval(function () { return __awaiter(_this, void 0, void 0, function () {
-        return __generator(this, function (_a) {
-            api.stop();
-            return [2 /*return*/];
-        });
-    }); }, 500);
-    console.log("Stopped");
-}
-function CheckIfClipsPlayAllTogetherConcurrently() {
-    var api = new API();
-    api.addPlayableClip(new Clip("clip1", 0, 1000));
-    api.addPlayableClip(new Clip("clip2", 0, 1000));
-    api.addPlayableClip(new Clip("clip3", 0, 1000));
-    api.addPlayableClip(new Clip("clip4", 0, 1000));
-    api.start();
-}
-function CheckIfClipsPlayAllTogether() {
-    var api = new API();
-    api.addPlayableClip(new Clip("clip1", 0, 1000));
-    api.addPlayableClip(new Clip("clip2", 0, 1000));
-    api.addPlayableClip(new Clip("clip3", 0, 1000));
-    api.addPlayableClip(new Clip("clip4", 0, 1000));
-    api.start();
-}
-CheckIfBasicClipWorks();
-// CheckIfTwoClipsAtTheSameTimeWorks()
-// CheckIfTwoClipsOneAfterAnotherWorks()
-// CheckIfTimeLineStopBeforeClipPlays()
-// CheckIfTimeLineStartAfterClipPlays()
-// CheckIfClipsPlayAllTogetherConcurrently()
-// CheckIfClipsPlayAllTogether()
+
+main()
+
+

test/test_bench.tsx

@@ -1,151 +0,0 @@
-// Write unsubcribe and run the remainder of the clip tech.
-// class TrackClip {
-//     name: string
-//     startOffset: number // ms in the 0 100 | 10 second clip
-//     timerID: NodeJS.Timeout | null
-//     lengthOfClip: number
-
-//     constructor(name: string, startOffset: number, lengthOfClip: number) {
-//         this.startOffset = startOffset
-//         this.lengthOfClip = lengthOfClip
-//         this.name = name
-//         this.timerID = null
-//     }
-
-//     async play(): Promise<void> {
-//         console.log(`Playing ${this.name}`)
-//         this.timerID = setInterval(async () => {
-//             console.log(`Running async ${this.name}`)
-//             this.stop()
-//         }, this.lengthOfClip)
-//     }
-
-//     async stop(): Promise<void> {
-//         clearInterval(this.timerID!)
-//         console.log(`Stopping ${this.name}`)
-//     }
-// }
-
-class TimeLine {
-    timelineItems: TrackClip[]
-    timerID: NodeJS.Timeout | null
-    timeLineLimit: number
-    
-    runningClips: TrackClip[]
-
-    constructor() {
-        this.timelineItems = []
-        this.timerID = null
-        this.timeLineLimit = 1000 * 10
-        this.runningClips = []
-    }
-
-    async addPlayableClip(clip: TrackClip): Promise<void> {
-        this.timelineItems.push(clip)
-    }
-
-    async play(): Promise<void> {
-        let start = 0
-        const updateInterval = 100
-        const timerID = setInterval(async () => {
-            console.log(`Current Time:${start}`)
-            
-            // get last updated time when stopped
-            for (const element of this.timelineItems) {
-                if (element.startOffset >= start) {
-                    element.play()
-                    // remove the element
-                    this.timelineItems = this.timelineItems.filter(item => item !== element)
-                }
-            }
-            if (start == this.timeLineLimit || this.timelineItems.length == 0) {
-                this.stop()
-            }
-            
-            start += updateInterval
-        }, updateInterval)
-        
-        this.timerID = timerID
-    }
-
-    async stop(): Promise<void> {
-        console.log(`Stopping Timeline`)
-        clearInterval(this.timerID!)
-    }
-}
-
-// How much timeline precision we have
-const percision = 100
-
-// Visual verification tests.
-function CheckIfBasicClipWorks() {
-    const api = new TimeLine()
-    api.addPlayableClip(new TrackClip("clip1", 0, 1000))
-    api.play()
-}
-
-function CheckIfBasicClipWorks3SecondsAfterTimelineStops() {
-    const api = new TimeLine()
-    api.addPlayableClip(new TrackClip("clip1", 0, 10000))
-    api.play()
-}
-
-function CheckIfTwoClipsAtTheSameTimeWorks() {
-    const api = new TimeLine()
-    api.addPlayableClip(new TrackClip("clip1", 0, 1000))
-    api.play()
-}
-
-function CheckIfTwoClipsOneAfterAnotherWorks() {
-    const api = new TimeLine()
-    api.addPlayableClip(new TrackClip("clip1", 0, 1000))
-    api.addPlayableClip(new TrackClip("clip2", 0, 2000))
-    api.play()
-}
-
-function CheckIfTimeLineStopBeforeClipPlays() {
-    const api = new TimeLine()
-    api.addPlayableClip(new TrackClip("clip3",0,1000))
-    api.play()
-    setInterval(async ()=> {
-        api.stop()
-    },1100)
-    console.log("Stopped")
-}
-
-function CheckIfTimeLineStartAfterClipPlays() {
-    const api = new TimeLine() 
-    api.addPlayableClip(new TrackClip("clip3",0,1000))
-    api.play()
-    setInterval(async ()=> {
-        api.stop()
-    },500)
-    console.log("Stopped")
-}
-
-function CheckIfClipsPlayAllTogetherConcurrently() {
-    const api = new TimeLine() 
-    api.addPlayableClip(new TrackClip("clip1",0,1000))
-    api.addPlayableClip(new TrackClip("clip2",0,1000))
-    api.addPlayableClip(new TrackClip("clip3",0,1000))
-    api.addPlayableClip(new TrackClip("clip4",0,1000))
-    api.play()
-}
-
-function CheckIfClipsPlayAllTogether() {
-    const api = new TimeLine() 
-    api.addPlayableClip(new TrackClip("clip1",0,1000))
-    api.addPlayableClip(new TrackClip("clip2",0,1000))
-    api.addPlayableClip(new TrackClip("clip3",0,1000))
-    api.addPlayableClip(new TrackClip("clip4",0,1000))
-    api.play()
-}
-
-CheckIfBasicClipWorks()
-//CheckIfTwoClipsAtTheSameTimeWorks()
-// CheckIfTwoClipsOneAfterAnotherWorks()
-// CheckIfTimeLineStopBeforeClipPlays()
-// CheckIfTimeLineStartAfterClipPlays()
-// CheckIfClipsPlayAllTogetherConcurrently()
-
-// CheckIfClipsPlayAllTogether()