Merge branch 'test' into feature/scene-object_control-panel

app/components/H4.tsx

@@ -1,19 +0,0 @@
-import { twMerge } from "tailwind-merge"
-
-export const H4 = ({
-  className,
-  children,
-}:{
-  className?: string,
-  children: React.ReactNode,
-})=>{
-  return(
-    <h4
-      className={
-        twMerge("text-base font-medium text-white", className)
-      }
-    >
-      {children}
-    </h4>
-  )
-}

app/components/Typography.tsx

@@ -0,0 +1,31 @@
+import { twMerge } from "tailwind-merge";
+
+interface TypogProps  {
+  className?: string;
+  children: React.ReactNode;
+}
+const baseTypo = "text-white";
+
+export const H1 = ({className,children,}:TypogProps)=>
+  <h1 className={twMerge(baseTypo, "font-medium text-2xl",className)}>{children}</h1>;
+
+export const H2 = ({className,children,}:TypogProps)=>
+  <h2 className={twMerge(baseTypo, "font-medium text-xl", className)}> {children} </h2>;
+
+export const H3 = ({className,children,}:TypogProps)=>
+  <h3 className={twMerge(baseTypo, "font-medium text-lg", className)}> {children} </h3>;
+
+export const H4 = ({className,children,}:TypogProps)=>
+  <h4 className={twMerge(baseTypo, "font-medium text-base", className)}> {children} </h4>;
+
+  export const Label = ({className,children,}:TypogProps)=>
+  <label className={twMerge(baseTypo, "font-medium text-base", className)}> {children} </label>;
+
+export const H5 = ({className,children,}:TypogProps)=>
+  <h5 className={twMerge(baseTypo, "font-medium text-sm",className)}> {children} </h5>;
+
+export const H6 = ({className,children,}:TypogProps)=>
+  <h6 className={twMerge(baseTypo, "font-light text-sm",className)}> {children} </h6>;
+
+export const P = ({className,children,}:TypogProps)=>
+  <p className={twMerge(baseTypo, className)}> {children} </p>;

app/components/index.tsx

@@ -4,7 +4,7 @@ import { ButtonLink } from "./ButtonLink";
 import { ButtonIconSelect } from "./ButtonIconSelect";
 import { Input } from "./Input";
 import { InputVector } from "./InputVector";
-import { H4 } from "./H4";
+import { H1, H2, H3, H4, H5, H6, P, Label} from "./Typography";
 import { ItemPicker } from "./ItemPicker";
 import { Textarea } from "./Textarea";
 import { LoadingBar } from "./LoadingBar";
@@ -15,7 +15,7 @@ export {
   ButtonIcon,
   ButtonLink,
   ButtonIconSelect,
-  H4,
+  H1,H2,H3,H4,H5,H6,P,Label,
   ItemPicker,
   Textarea,
   Input,

app/modules/SidePanel/SidePanel.tsx

@@ -27,7 +27,6 @@ export const SidePanel = ({ children }: Props) => {
       <div className="relative w-[23.5rem] h-full">
         {children}
       </div>
-
     </div>
   );
 };

app/pages/PageEnigma/PageEnigmaComponent.tsx

@@ -104,4 +104,4 @@ export const PageEnigmaComponent = () => {
       </AppUIProvider>
     </div>
   );
-};
+};

app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx

@@ -70,15 +70,15 @@ export const ControlsTopButtons = () => {
     editorEngine?.switchCameraView();
   };
   const handleButtonPlayBack = () => {
-    editorEngine?.start_playback();
+    editorEngine?.startPlayback();
   };
   const handleButtonRender = () => {
-    editorEngine?.take_timeline_cam_clip();
-  };
-  const handleButtonPlay = () => {
-    editorEngine?.start_playback();
+    editorEngine?.generateVideo();
   };
 
+  const handleButtonTakeFrame = () => {
+    editorEngine?.take_timeline_cam_clip();
+  };
 
   const handleButtonLoad = () => {};
 
@@ -148,9 +148,11 @@ export const ControlsTopButtons = () => {
       </div>
       <div className="flex gap-2">
 
-        <Button onClick={handleButtonLoad}>Load</Button>
-        <Button onClick={handleButtonRender}>Take Frame</Button>
-        <Button onClick={handleButtonPlay}>Play</Button>
+        <Button onClick={handleButtonLoad}>Test Button</Button>
+        <Button onClick={handleButtonTakeFrame}>Take Frame</Button>
+        <Button onClick={handleButtonRender}>Render</Button>
+        {/* <Button onClick={handleButtonLoad}>Load</Button>
+        <Button onClick={handleButtonRender}>Render</Button> */}
         <Button
           onClick={handleButtonTest}
           className="bg-brand-tertiary hover:bg-brand-teriary-400 focus-visible:outline-brand-tertiary"

app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx

@@ -1,3 +1,5 @@
+import { useContext } from "react";
+
 import {
   faBackwardFast,
   faBackwardStep,
@@ -5,9 +7,12 @@ import {
   faForwardStep,
   faPlay,
 } from "@fortawesome/pro-solid-svg-icons";
+import { EngineContext } from "../../contexts/EngineContext";
 import { ButtonIcon } from "~/components";
 
 export const ControlsVideo = () => {
+  const editorEngine = useContext(EngineContext);
+  
   const handleBackwardFast = () => {
     console.log("Controls Video: Backward-Fast clicked");
   };
@@ -16,6 +21,8 @@ export const ControlsVideo = () => {
   };
   const handlePlay = () => {
     console.log("Controls Video: Play clicked");
+    editorEngine?.startPlayback();
+    
   };
   const handleForwardStep = () => {
     console.log("Controls Video: Forward-Step clicked");

app/pages/PageEnigma/comps/Helmet.tsx

@@ -0,0 +1,24 @@
+import { useContext, useEffect, useState } from "react";
+import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
+import { TopBarHelmet } from "~/modules/TopBarHelmet/TopBarHelmet";
+import { Button } from "~/components";
+import { EngineContext } from "../contexts/EngineContext";
+
+export const Helmet = () => {
+  const [helmetKey, setHelmetKey] = useState<number>(Date.now());
+  const engineContext = useContext(EngineContext);
+  
+  const handleGenerateVideo = () => {
+    engineContext?.generateVideo();
+  };
+  
+  useEffect(()=>{
+    setHelmetKey(Date.now());
+  },[engineContext]);
+
+  return (
+    <TopBarHelmet key={helmetKey}>
+      <Button icon={faSparkles} onClick={handleGenerateVideo}>Generate Movie</Button>
+    </TopBarHelmet>
+  );
+}

app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx

@@ -8,7 +8,7 @@ export const PreviewEngineCamera = ()=>{
 
   const [showLoader, setShowLoader] = useState<boolean>(true);
   useEffect(()=>{
-    setTimeout(()=>setShowLoader(false), 5000);
+    setTimeout(()=>setShowLoader(false), 500);
   },[]);
   return (
     <div

app/pages/PageEnigma/comps/Timeline/Timeline.tsx

@@ -17,8 +17,11 @@ import {
   timelineHeight,
 } from "~/pages/PageEnigma/store";
 import { useQueueHandler } from "~/pages/PageEnigma/comps/Timeline/utils/useQueueHandler";
+import { EngineContext } from "../../contexts/EngineContext";
 
 export const Timeline = () => {
+  const editorEngine = useContext(EngineContext);
+    
   const {
     characters,
     objects,

app/pages/PageEnigma/contexts/EngineProvider.tsx

@@ -4,7 +4,7 @@ import Editor from "~/pages/PageEnigma/js/editor";
 
 import { EngineContext } from "./EngineContext";
 import { AppUiContext } from "./AppUiContext";
-
+import { TrackContext } from "./TrackContext/TrackContext";
 
 interface Props {
   children: ReactNode;
@@ -13,6 +13,8 @@ interface Props {
 export const EngineProvider = ({ children }: Props) => {
   const [editor, setEditor] = useState<Editor | null>(null);
   const [appUiState, dispatchAppUiState] = useContext(AppUiContext);
+  // To talk to react land.
+  const trackContext = useContext(TrackContext)
 
   useEffect(() => {
     //componentDidMount

app/pages/PageEnigma/contexts/TrackContext/TrackProvider.tsx

@@ -76,8 +76,26 @@ export const TrackProvider = ({ children }: Props) => {
       {
         id: "ani-id1",
         offset: 0,
-        length: 124,
-        name: "ani 11",
+        length: 25,
+        name: "Sit",
+      },
+      {
+        id: "ani-id2",
+        offset: 0,
+        length: 25,
+        name: "Idle",
+      },
+      {
+        id: "ani-id3",
+        offset: 0,
+        length: 25,
+        name: "Stand",
+      },
+      {
+        id: "ani-id4",
+        offset: 0,
+        length: 25,
+        name: "Walk",
       },
     ]);
     setAudioClips([]);

app/pages/PageEnigma/datastructures/clips/animation_clip.ts

@@ -74,19 +74,16 @@ export class AnimationClip  {
           this.clip_action.play();
         }
       }
-      console.log("Play")
     }
   }
 
   step(deltatime: number) {
     if (this.mixer == null) { return; }
     this.mixer?.update(deltatime);
-    console.log("Update")
   }
 
   stop() {
     this.mixer?.stopAllAction();
-    console.log("stop.")
   }
 
   toJSON(): string {

app/pages/PageEnigma/datastructures/clips/transform_clip.ts

@@ -29,10 +29,11 @@ export class TransformClip {
     this.looping = false;
   }
 
-  step(object: THREE.Object3D) {
-    //if (this.step_frame >= 60/this.length && this.looping == false) { return; } // Reached max frames.
+  step(object: THREE.Object3D, offset: number, frame: number) {
+    this.step_frame = frame-offset;
+    if (this.step_frame < 0) { this.step_frame = 0; }
+    if (this.step_frame >= this.length) { return; } // Reached max frames.
     if(this.positions.length < 2) { return; } // If there are enough points in the scene.
-    this.step_frame += 1;
 
     let time_frame = (this.step_frame/this.length);
 

app/pages/PageEnigma/js/api_manager.ts

@@ -8,6 +8,21 @@ import { TimelineDataState } from "./timeline";
  * Storyteller Studio API Manager
  * The source of truth of all these media items is the database in the cloud
  */
+class MediaFile {
+  public_bucket_path: string;
+  media_type: string;
+  media_token: string;
+
+  constructor(
+    public_bucket_path: string,
+    media_type: string,
+    media_token: string,
+  ) {
+    this.media_token = media_token;
+    this.public_bucket_path = public_bucket_path;
+    this.media_type = media_type;
+  }
+}
 
 /**
  * This is designed to surface user customer facing messages as errors.
@@ -61,7 +76,8 @@ class APIManager {
       file,
       scene_glb_media_file_token,
     );
-    const result_scene_glb_media_file_token = upload_glb_response["media_file_token"];
+    const result_scene_glb_media_file_token =
+      upload_glb_response["media_file_token"];
 
     // now write the scene
     const save_scene_timeline_response = await this.saveSceneAndTimeLineState(
@@ -74,12 +90,16 @@ class APIManager {
     const result_scene_media_file_token =
       save_scene_timeline_response["media_file_token"];
 
-    const data = { "scene_glb_media_file_token":result_scene_glb_media_file_token,
-                    "scene_media_file_token": result_scene_media_file_token };
+    const data = {
+      scene_glb_media_file_token: result_scene_glb_media_file_token,
+      scene_media_file_token: result_scene_media_file_token,
+    };
     return new APIManagerResponseSuccess("Scene Saved", data);
   }
 
-  public async loadSceneState(scene_media_file_token: string | null): Promise<APIManagerResponseSuccess> {
+  public async loadSceneState(
+    scene_media_file_token: string | null,
+  ): Promise<APIManagerResponseSuccess> {
     const api_base_url = "https://api.fakeyou.com";
     const url = `${api_base_url}/v1/media_files/file/${scene_media_file_token}`;
     const response = await fetch(url);
@@ -104,19 +124,28 @@ class APIManager {
 
     console.log(`loadSceneState: ${JSON.stringify(json_result)}`);
 
-    const scene_glb_media_file_token: string = json_result["scene_glb_media_file_token"];
-    
-    const media_bucket_path = await this.getMediaFile(scene_glb_media_file_token);
-    console.log(`GLB ${media_bucket_path}`)
+    const scene_glb_media_file_token: string =
+      json_result["scene_glb_media_file_token"];
+
+    const media_bucket_path = await this.getMediaFile(
+      scene_glb_media_file_token,
+    );
+    console.log(`GLB ${media_bucket_path}`);
     let glbLoader = new GLTFLoader();
     // promisify this
-    const loadGlb = (bucket_path: string): Promise<APIManagerResponseSuccess> => {
+    const loadGlb = (
+      bucket_path: string,
+    ): Promise<APIManagerResponseSuccess> => {
       return new Promise((resolve, reject) => {
         glbLoader.load(bucket_path, (glb) => {
           if (glb) {
             const scene: THREE.Scene = glb.scene;
-            const data = { "scene_glb_media_file_token": scene_glb_media_file_token , "scene_media_file_token":scene_media_file_token, "scene": scene };
-            console.log(`Data: ${data}`)
+            const data = {
+              scene_glb_media_file_token: scene_glb_media_file_token,
+              scene_media_file_token: scene_media_file_token,
+              scene: scene,
+            };
+            console.log(`Data: ${data}`);
             resolve(new APIManagerResponseSuccess("Success Loaded", data));
           } else {
             throw new APIManagerResponseError("Failed to Load GLB Scene");
@@ -133,7 +162,7 @@ class APIManager {
    * @param media_file_token
    * @returns
    */
-  private async getMediaFile(media_file_token: string): Promise<string> {
+  public async getMediaFile(media_file_token: string): Promise<string> {
     let api_base_url = "https://api.fakeyou.com";
     let url = `${api_base_url}/v1/media_files/file/${media_file_token}`;
     let response = await fetch(url);
@@ -205,12 +234,14 @@ class APIManager {
     const url = `${this.baseUrl}/v1/media_files/write/scene_file`;
     let uuid = uuidv4();
 
-    console.log(`Saving Scene scene_media_file_token:${scene_media_file_token} | scene_glb_media_file_token:${scene_glb_media_file_token}`)
+    console.log(
+      `Saving Scene scene_media_file_token:${scene_media_file_token} | scene_glb_media_file_token:${scene_glb_media_file_token}`,
+    );
     // turn json into a blob
     const scene_schema = {
-      'scene_glb_media_file_token': scene_glb_media_file_token,
-      'scene_name': scene_file_name,
-      'timeline': { 'objects': [] },
+      scene_glb_media_file_token: scene_glb_media_file_token,
+      scene_name: scene_file_name,
+      timeline: { objects: [] },
     };
     const json = JSON.stringify(scene_schema);
     const blob = new Blob([json], { type: "application/json" });
@@ -282,10 +313,25 @@ class APIManager {
     }
   }
 
-  public async batchGetMedia(media_tokens: []): Promise<string> {
-    return "";
+  public async getMediaBatch(media_tokens: string[]): Promise<MediaFile[]> {
+    const tokens = media_tokens;
+    const url = new URL(`${this.baseUrl}/v1/media_files/batch`);
+    tokens.forEach((token) => url.searchParams.append("tokens", token));
+    const result = await fetch(url)
+      .then((response) => response.json())
+      .then((data) => {
+        const result = data["media_files"].map((element) => {
+          return new MediaFile(
+            element["public_bucket_path"],
+            element["media_type"],
+            element["token"],
+          );
+        });
+        return result;
+      })
+      .catch((error) => console.error("Error:", error));
+    return result;
   }
-
 }
 
 export default APIManager;

app/pages/PageEnigma/js/editor.ts

@@ -1,6 +1,6 @@
 import * as THREE from "three";
 import { OrbitControls } from "three/addons/controls/OrbitControls.js";
-import { FreeCam } from "./free_cam"
+import { FreeCam } from "./free_cam";
 import { TransformControls } from "three/addons/controls/TransformControls.js";
 import Scene from "./scene.js";
 import APIManager from "./api_manager.js";
@@ -12,7 +12,7 @@ import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js";
 import { SAOPass } from "three/addons/postprocessing/SAOPass.js";
 import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js";
 import { BokehPass } from "three/addons/postprocessing/BokehPass.js";
-import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg";
+import { FFmpeg, createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg";
 import AudioEngine from "./audio_engine.js";
 import TransformEngine from "./transform_engine.js";
 import { TimeLine, TimelineDataState } from "./timeline.js";
@@ -20,10 +20,9 @@ import { ClipUI } from "../datastructures/clips/clip_offset.js";
 import { LipSyncEngine } from "./lip_sync_engine.js";
 import { AnimationEngine } from "./animation_engine.js";
 
-import { PointerLockControls } from 'three/addons/controls/PointerLockControls.js';
+import { PointerLockControls } from "three/addons/controls/PointerLockControls.js";
 import { APPUI_ACTION_TYPES } from "../reducers";
 
-
 class EditorState {
   // {
   //   action: "ShowLoadingIndicator"
@@ -44,6 +43,7 @@ class Editor {
   version: number;
   activeScene: Scene;
   camera: any;
+  render_camera: any;
   renderer: THREE.WebGLRenderer | undefined;
   clock: THREE.Clock | undefined;
   canvReference: any;
@@ -87,12 +87,14 @@ class Editor {
   cam_obj: THREE.Object3D | undefined;
   renderPass: RenderPass | undefined;
 
+  camera_person_mode: boolean;
   current_scene_media_token: string | null;
   current_scene_glb_media_token: string | null;
 
   can_initialize: boolean;
   dispatchAppUiState: any; // todo figure out the type
-  camera_person_mode: boolean;
+  render_width: number;
+  render_height: number;
 
   // Default params.
   constructor() {
@@ -116,6 +118,7 @@ class Editor {
     this.activeScene = new Scene("" + this.version);
     this.activeScene.initialize();
     this.camera;
+    this.render_camera;
     this.renderer;
     this.clock;
     this.canvReference = null;
@@ -159,6 +162,9 @@ class Editor {
     this.max_length = 10;
     // Audio Engine Test.
 
+    this.render_width = 1280;
+    this.render_height = 720;
+
     this.audio_engine = new AudioEngine();
     this.transform_engine = new TransformEngine(this.version);
     this.lipsync_engine = new LipSyncEngine();
@@ -174,7 +180,7 @@ class Editor {
 
     this.current_frame = 0;
 
-    // Dispatcher 
+    // Dispatcher
     this.dispatchAppUiState = null;
 
     // Scene State
@@ -182,19 +188,17 @@ class Editor {
     this.current_scene_glb_media_token = null;
   }
 
-  initialize(config:any) {
+  initialize(config: any) {
     //setup reactland Callbacks
-    this.dispatchAppUiState = config.dispatchAppUiState
+    this.dispatchAppUiState = config.dispatchAppUiState;
 
-    // Show Loader
-    this.dispatchAppUiState({
-      type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER
-    });
+    // this is called by the parent for some reason
+    // this.dispatchAppUiState({
+    //   type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER
+    // });
 
     if (this.can_initialize == false) {
-      console.log(
-        "Editor Already Initialized",
-      );
+      console.log("Editor Already Initialized");
       return;
     }
     this.can_initialize = false;
@@ -209,6 +213,9 @@ class Editor {
     this.camera.position.z = 3;
     this.camera.position.y = 3;
     this.camera.position.x = -3;
+
+    this.render_camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50);
+
     // Base WebGL render and clock for delta time.
     this.renderer = new THREE.WebGLRenderer({
       antialias: true,
@@ -225,8 +232,14 @@ class Editor {
     this._configure_post_pro();
     // Controls and movement.
 
-    this.lockControls = new PointerLockControls(this.camera, this.renderer.domElement);
-    this.cameraViewControls = new FreeCam(this.camera, this.renderer.domElement);
+    this.lockControls = new PointerLockControls(
+      this.camera,
+      this.renderer.domElement,
+    );
+    this.cameraViewControls = new FreeCam(
+      this.camera,
+      this.renderer.domElement,
+    );
     this.cameraViewControls.movementSpeed = 1;
     this.cameraViewControls.domElement = this.renderer.domElement;
     this.cameraViewControls.rollSpeed = Math.PI / 24;
@@ -234,14 +247,17 @@ class Editor {
     this.cameraViewControls.dragToLook = true;
     this.cameraViewControls.enabled = false;
 
-    this.orbitControls = new OrbitControls(this.camera, this.renderer.domElement);
+    this.orbitControls = new OrbitControls(
+      this.camera,
+      this.renderer.domElement,
+    );
 
     this.control = new TransformControls(this.camera, this.renderer.domElement);
     // OnClick and MouseMove events.
     window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
     window.addEventListener("click", this.onMouseClick.bind(this), false);
     // Base control and debug stuff remove debug in prod.
-    this._initialize_control();
+    this._initializeControl();
     // Resets canvas size.
     this.onWindowResize();
     // Creates the main update loop.
@@ -251,32 +267,63 @@ class Editor {
 
     this._test_demo();
 
-    // saving state of the scene
-    this.current_scene_media_token = null;
-    this.current_scene_glb_media_token = null;
-
 
     this.renderer.domElement.addEventListener("mousedown", this.onMouseDown.bind(this), false);
     this.renderer.domElement.addEventListener("mouseup", this.onMouseUp.bind(this), false);
     this.renderer.domElement.addEventListener("onContextMenu", this.onContextMenu.bind(this), false);
-  
+
     this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
-    if(this.cam_obj)
-    {
+    if (this.cam_obj) {
       this.add_transform_clip_base("Camera Object", this.cam_obj, 0, 150)
     }
 
-    // hide loader
+    // saving state of the scene
+    this.current_scene_media_token = null;
+    this.current_scene_glb_media_token = null;
+
+    this.renderer.domElement.addEventListener(
+      "mousedown",
+      this.onMouseDown.bind(this),
+      false,
+    );
+    this.renderer.domElement.addEventListener(
+      "mouseup",
+      this.onMouseUp.bind(this),
+      false,
+    );
+    this.renderer.domElement.addEventListener(
+      "onContextMenu",
+      this.onContextMenu.bind(this),
+      false,
+    );
+
+    this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
+    if (this.cam_obj) {
+      this.add_transform_clip_base("Camera Object", this.cam_obj, 0, 150);
+    }
+
     this.dispatchAppUiState({
-      type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER
+      type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR,
+      payload: {
+        showEditorLoadingBar: {
+          progress: 100,
+        },
+      },
     });
 
+    this.dispatchAppUiState({
+      type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADINGBAR,
+    });
   }
 
   // Token comes in from the front end to load the scene from the site.
+  public async testBatchRequest() {
+    const result = await this.api_manager.getMediaBatch(["m_8fmp9hrvsqcryzka1fra597kg42s50","m_z4jzbst3xfh64h0qn4bqh4afenfps9"]);
+    console.log(result);
+  }
   public async loadScene(scene_media_token: string) {
     this.dispatchAppUiState({
-      type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER
+      type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
     });
 
     if (scene_media_token != null) {
@@ -287,19 +334,25 @@ class Editor {
       this.current_scene_media_token,
     );
 
-    console.log(load_scene_state_response)
-    if(load_scene_state_response.data == null) { 
-      console.log("load_scene_state_response Missing Data")
-      return; }
+    console.log(load_scene_state_response);
+    if (load_scene_state_response.data == null) {
+      console.log("load_scene_state_response Missing Data");
+      return;
+    }
 
-    const loaded_scene = load_scene_state_response.data["scene"]
+    const loaded_scene = load_scene_state_response.data["scene"];
 
+    if (load_scene_state_response.data == null) { return; }
     // Load these so you can rewrite the scene glb using it's token.
-    this.current_scene_media_token = load_scene_state_response.data["scene_media_file_token"]
-    this.current_scene_glb_media_token = load_scene_state_response.data["scene_glb_media_file_token"]
+    this.current_scene_media_token =
+      load_scene_state_response.data["scene_media_file_token"];
+    this.current_scene_glb_media_token =
+      load_scene_state_response.data["scene_glb_media_file_token"];
+
+    console.log(
+      `loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
+    );
 
-    console.log(`loadScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`);
-    
     this.activeScene.scene.children = loaded_scene.children;
 
     this.activeScene.scene.children.forEach((child: THREE.Object3D) => {
@@ -316,53 +369,59 @@ class Editor {
     });
 
     this.dispatchAppUiState({
-      type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER
+      type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
     });
   }
 
-
-
   public async saveScene(name: string) {
     // remove controls when saving scene.
     this.removeTransformControls();
     this.dispatchAppUiState({
-      type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER
+      type: APPUI_ACTION_TYPES.SHOW_EDITOR_LOADER,
     });
-    console.log(`saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`);
-    
+    console.log(
+      `saveScene => SceneMediaToken:${this.current_scene_media_token} SceneGLBMediaToken:${this.current_scene_glb_media_token}`,
+    );
+
     const result = await this.api_manager.saveSceneState(
       this.activeScene.scene,
       name,
       this.current_scene_glb_media_token,
       this.current_scene_media_token,
-      new TimelineDataState()
-    )
+      new TimelineDataState(),
+    );
 
-    if(result.data == null) { return; }
-
-    const scene_media_token = result.data["scene_media_file_token"]
-    if (scene_media_token != null) {
-      this.current_scene_media_token = scene_media_token
+    if (result.data == null) {
+      return;
     }
-    const scene_glb_media_token =  result.data["scene_glb_media_file_token"]
+
+    const scene_media_token = result.data["scene_media_file_token"];
+    if (scene_media_token != null) {
+      this.current_scene_media_token = scene_media_token;
+    }
+    const scene_glb_media_token = result.data["scene_glb_media_file_token"];
     if (scene_glb_media_token != null) {
-      this.current_scene_glb_media_token = scene_glb_media_token
-    } 
+      this.current_scene_glb_media_token = scene_glb_media_token;
+    }
 
     this.dispatchAppUiState({
-        type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER
+      type: APPUI_ACTION_TYPES.HIDE_EDITOR_LOADER,
     });
   }
 
   /**
-   * This cleans up the transform controls 
-   * During saving it 
+   * This cleans up the transform controls
+   * During saving it
    * Doesn't retain those controls.
-   * @returns 
+   * @returns
    */
   private removeTransformControls() {
-    if (this.control == undefined) { return };
-    if (this.outlinePass == undefined) { return };
+    if (this.control == undefined) {
+      return;
+    }
+    if (this.outlinePass == undefined) {
+      return;
+    }
     this.last_selected = this.selected;
     this.control.detach();
     this.activeScene.scene.remove(this.control);
@@ -422,29 +481,41 @@ class Editor {
   switchCameraView() {
     this.camera_person_mode = !this.camera_person_mode;
     if (this.cam_obj) {
-      if(this.camera_person_mode) {
+      if (this.camera_person_mode) {
         this.last_cam_pos.copy(this.camera.position);
         this.last_cam_rot.copy(this.camera.rotation);
 
         this.camera.position.copy(this.cam_obj.position);
         this.camera.rotation.copy(this.cam_obj.rotation);
-        if(this.orbitControls) { this.orbitControls.enabled = false; }
-        if(this.lockControls) { this.activeScene.scene.add(this.lockControls.getObject()); }
-        if(this.cameraViewControls) { this.cameraViewControls.enabled = true; }
+        if (this.orbitControls) {
+          this.orbitControls.enabled = false;
+        }
+        if (this.lockControls) {
+          this.activeScene.scene.add(this.lockControls.getObject());
+        }
+        if (this.cameraViewControls) {
+          this.cameraViewControls.enabled = true;
+        }
 
-        if(this.activeScene.hot_items){
-          this.activeScene.hot_items.forEach(element => {
+        if (this.activeScene.hot_items) {
+          this.activeScene.hot_items.forEach((element) => {
             element.visible = false;
           });
         }
       } else {
         this.camera.position.copy(this.last_cam_pos);
         this.camera.rotation.copy(this.last_cam_rot);
-        if(this.orbitControls) { this.orbitControls.enabled = true; }
-        if(this.lockControls) { this.activeScene.scene.remove(this.lockControls.getObject()); }
-        if(this.cameraViewControls) { this.cameraViewControls.enabled = false; }
-        if(this.activeScene.hot_items){
-          this.activeScene.hot_items.forEach(element => {
+        if (this.orbitControls) {
+          this.orbitControls.enabled = true;
+        }
+        if (this.lockControls) {
+          this.activeScene.scene.remove(this.lockControls.getObject());
+        }
+        if (this.cameraViewControls) {
+          this.cameraViewControls.enabled = false;
+        }
+        if (this.activeScene.hot_items) {
+          this.activeScene.hot_items.forEach((element) => {
             element.visible = true;
           });
         }
@@ -452,9 +523,22 @@ class Editor {
     }
   }
 
-  async add_transform_clip_base(name: string = "New Clip", object: THREE.Object3D, start_offset:number, end_offset:number) {
+  async add_transform_clip_base(
+    name: string = "New Clip",
+    object: THREE.Object3D,
+    start_offset: number,
+    end_offset: number,
+  ) {
     this.timeline.addPlayableClip(
-      new ClipUI(1.0, "transform", "clip2", object.uuid, object.uuid, start_offset, end_offset),
+      new ClipUI(
+        1.0,
+        "transform",
+        "clip2",
+        object.uuid,
+        object.uuid,
+        start_offset,
+        end_offset,
+      ),
     );
     this.transform_engine.loadObject(object.uuid, end_offset);
   }
@@ -510,6 +594,18 @@ class Editor {
         400,
       ),
     );
+
+
+    this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v");
+
+    await this.timeline.addPlayableClip(new ClipUI(
+      1.0,
+      "audio",
+      "AudioClip",
+      "m_h33vytxs5eqqqf07nsy14qzrf9ww4v",
+      "",
+      0,
+      50));
   }
 
   // Configure post processing.
@@ -572,8 +668,8 @@ class Editor {
     this.composer.addPass(this.smaaPass);
     this.composer.addPass(this.bokehPass);
 
-    this.outputPass = new OutputPass()
-    this.composer.addPass(this.outputPass)
+    this.outputPass = new OutputPass();
+    this.composer.addPass(this.outputPass);
   }
 
   create_parim(name: string) {
@@ -583,6 +679,7 @@ class Editor {
   render_mode() {
     this.rendering = !this.rendering;
     this.activeScene.render_mode(this.rendering);
+    console.log("works.")
 
     //if (this.rendering) {
     //    this._remove_post_processing()
@@ -628,9 +725,16 @@ class Editor {
   }
 
   take_timeline_cam_clip() {
-    if(this.cam_obj == null) { return; }
-    if(!this.camera_person_mode) { return; }
-    this.transform_engine.addFrame(this.cam_obj, this.transform_engine.clips[this.cam_obj.uuid].length);
+    if (this.cam_obj == null) {
+      return;
+    }
+    if (!this.camera_person_mode) {
+      return;
+    }
+    this.transform_engine.addFrame(
+      this.cam_obj,
+      this.transform_engine.clips[this.cam_obj.uuid].length,
+    );
     console.log("Camera frame taken.");
     this.activeScene.createPoint(this.cam_obj.position, false);
   }
@@ -649,7 +753,7 @@ class Editor {
     let delta_time = this.clock.getDelta();
 
     if (this.cameraViewControls && this.camera_person_mode) {
-      this.cameraViewControls.update(5 * delta_time)
+      this.cameraViewControls.update(5 * delta_time);
       if (this.cam_obj) {
         if (this.timeline.isPlaying == false) {
           this.cam_obj.position.copy(this.camera.position);
@@ -661,19 +765,14 @@ class Editor {
       }
     }
 
+    if (this.render_camera && this.cam_obj) {
+      this.render_camera.position.copy(this.cam_obj.position);
+      this.render_camera.rotation.copy(this.cam_obj.rotation);
+    }
+
     this.timeline.update(delta_time);
 
-    this.render_scene();
-    if (this.capturer != null) {
-      this.capturer.capture(this.renderer.domElement);
-    } // Record scene.
-  }
-
-  start_playback() {
-    this.timeline.isPlaying = true;
-    if(!this.camera_person_mode) {
-      this.switchCameraView();
-    }
+    this.renderScene();
   }
 
   change_mode(type: any) {
@@ -707,6 +806,56 @@ class Editor {
     return blob;
   }
 
+  async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) {
+    let video_og = itteration + 'tmp.mp4';
+    let wav_name = itteration + 'tmp.wav';
+    let new_video = (itteration + 1) + 'tmp.mp4';
+    let startFrame = clip.start_offset;
+    let endFrame = clip.ending_offset;
+
+
+    if (endFrame > this.timeline.timeLineLimit) {
+      endFrame = this.timeline.timeLineLimit;
+    }
+    if (startFrame > this.timeline.timeLineLimit) {
+      startFrame = this.timeline.timeLineLimit-1;
+    }
+
+    const startTime = startFrame / this.cap_fps;
+    const endTime = endFrame / this.cap_fps;
+    let end = endTime - startTime;
+
+    let duration = this.timeline.timeLineLimit / this.cap_fps;
+
+    let audioSegment = "as_" + wav_name;
+    await ffmpeg.FS('writeFile', wav_name, await fetchFile(await this.api_manager.getMediaFile(clip.media_id)));
+    await ffmpeg.run('-i',
+      wav_name,
+      '-ss', '0',
+      '-to', '' + end,
+      "-max_muxing_queue_size", "999999",
+      audioSegment);
+
+    await ffmpeg.run('-i', video_og,"-max_muxing_queue_size", "999999", `${itteration}empty_tmp.wav`);
+
+    await ffmpeg.run(
+      '-i', `${itteration}empty_tmp.wav`, 
+      '-i', audioSegment, 
+      '-filter_complex', "[1:a]adelay="+startTime*1000+"|"+startTime*1000+"[a1];[0:a][a1]amix=inputs=2[a]",
+      "-map", "[a]",
+      `${itteration}final_tmp.wav`)
+
+    await ffmpeg.run(
+      '-i', video_og, 
+      '-i', `${itteration}final_tmp.wav`, 
+      '-c:v', 'copy',
+      '-c:a', 'aac',
+      '-map', '0:v:0',
+      '-map', '1:a:0',
+      '-strict', 'experimental',
+      new_video)
+  }
+
   async stopPlayback() {
     this.render_mode();
     let ffmpeg = createFFmpeg({ log: true });
@@ -724,35 +873,78 @@ class Editor {
       "" + this.cap_fps,
       "-i",
       "image%d.png",
-      "output.mp4",
+      "-f",
+      "lavfi",
+      "-i",
+      "anullsrc", // This adds a silent audio track
+      "-max_muxing_queue_size",
+      "999999",
+      "-c:v",
+      "libx264", // Specify video codec (optional, but recommended for MP4)
+      "-c:a",
+      "aac", // Specify audio codec (optional, but recommended for MP4)
+      "-shortest", // Ensure output duration matches the shortest stream (video or audio)
+      "0tmp.mp4"
     );
-    let output = await ffmpeg.FS("readFile", "output.mp4");
+
+
+    let itteration = 0;
+
+    for (const clip of this.timeline.timelineItems) {
+      if (clip.type == "lipsync" || clip.type == "audio") {
+        await this.convertAudioClip(itteration, ffmpeg, clip);
+        itteration += 1;
+      }
+    };
+
+    let output = await ffmpeg.FS("readFile", itteration + "tmp.mp4");
     // Create a Blob from the output file for downloading
     const blob = new Blob([output.buffer], { type: "video/mp4" });
     const url = URL.createObjectURL(blob);
-    await this.api_manager.uploadMedia(blob, "output.mp4");
-    // Create a link to download the file
     const downloadLink = document.createElement("a");
     downloadLink.href = url;
-    downloadLink.download = "output.mp4";
+    downloadLink.download = "render.mp4";
     document.body.appendChild(downloadLink);
     downloadLink.click();
     // Clean up
     URL.revokeObjectURL(url);
     document.body.removeChild(downloadLink);
+  
+  
+    let data = await this.api_manager.uploadMedia(blob, "tmp.wav");
+    console.log(data);
+    // Create a link to download the file
+
+  }
+
+  generateVideo() {
+    console.log("Generating video...");
+    if (this.rendering) { return; }
+    this.startPlayback();
+    this._initializeRecording();
+    this.rendering = true;
+    this.activeScene.render_mode(this.rendering);
+    if (this.activeScene.hot_items) {
+      this.activeScene.hot_items.forEach(element => {
+        element.visible = false;
+      });
+    }
   }
 
   startPlayback() {
-    this.playback_location = 0;
-    this._initialize_recording();
+    this.timeline.isPlaying = true;
+    this.timeline.scrubberPosition = 0;
+    if (!this.camera_person_mode) {
+      this.switchCameraView();
+    }
   }
 
   // Initializes transform x y z changes.
-  _initialize_control() {
+  _initializeControl() {
     if (this.control == undefined) {
       return;
     }
-    this.control.addEventListener("change", this.render_scene.bind(this));
+    this.control.addEventListener("change", this.renderScene.bind(this));
     this.control.addEventListener("dragging-changed", (event: any) => {
       if (this.orbitControls == undefined) {
         return;
@@ -766,18 +958,18 @@ class Editor {
     this.activeScene.scene.add(this.control);
   }
 
-  // Initializes CCapture for capturing the scene to send over to backend.
-  _initialize_recording() {
+  _initializeRecording() {
     this.frame_buffer = [];
     this.render_timer = 0;
   }
 
   // Render the scene to the camera.
-  render_scene() {
-    if (this.composer != null) {
+  renderScene() {
+    if (this.composer != null && !this.rendering) {
       this.composer.render();
-    } else if (this.renderer && this.camera) {
-      this.renderer.render(this.activeScene.scene, this.camera);
+    } else if (this.renderer && this.render_camera) {
+      this.renderer.setSize(this.render_width, this.render_height);
+      this.renderer.render(this.activeScene.scene, this.render_camera);
     } else {
       console.error("Could not render to canvas no render or composer!");
     }
@@ -787,10 +979,11 @@ class Editor {
       let imgData = this.renderer.domElement.toDataURL();
       this.frame_buffer.push(imgData);
       this.render_timer += this.clock.getDelta();
-      if (this.playback_location >= this.fps_number * 3) {
+      if (this.timeline.isPlaying == false) {
         this.stopPlayback();
         this.playback_location = 0;
         this.rendering = false;
+        this.onWindowResize();
       }
     }
   }
@@ -813,6 +1006,12 @@ class Editor {
     if (this.composer != null) {
       this.composer.setSize(width, height);
     }
+
+    if (this.render_camera == undefined) { return; }
+
+    //this.renderer.setSize(this.render_width, this.render_height);
+    this.render_camera.aspect = this.render_width / this.render_height;
+    this.render_camera.updateProjectionMatrix();
   }
 
   onContextMenu(event: any) {

app/pages/PageEnigma/js/timeline.ts

@@ -72,7 +72,7 @@ export class TimeLine {
         scene: Scene) {
         
         this.timelineItems = []
-        this.timeLineLimit = 60 * 10 // 10 seconds
+        this.timeLineLimit = 60 * 5 // 5 seconds
      
         this.isPlaying = false
         this.scrubberPosition = 0 // in frames into the tl
@@ -151,7 +151,7 @@ export class TimeLine {
             else if (element.type == "animation") {
             }
             else if (element.type == "lipsync") {
-                
+                this.lipSyncEngine.clips[element.object_uuid].reset();
             }
             else {
                 this.stop()
@@ -179,13 +179,17 @@ export class TimeLine {
                 let object = this.scene.get_object_by_uuid(element.object_uuid)
                 if (element.type == "transform") {
                     if (object && this.transformEngine.clips[element.object_uuid]) {
-                        this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset)
-                        this.transformEngine.clips[element.object_uuid].step(object)
+                        this.transformEngine.clips[element.object_uuid].length = (element.ending_offset - element.start_offset);
+                        this.transformEngine.clips[element.object_uuid].step(object, element.start_offset, this.scrubberPosition);
                     }
                 }
                 else if (element.type == "audio") {
-                    // global audio track
-                    this.audioEngine.playClip(element.media_id)
+                    if(this.scrubberPosition+1 >= element.ending_offset){
+                        this.audioEngine.stopClip(element.media_id);
+                    }
+                    else {
+                        this.audioEngine.playClip(element.media_id);
+                    }
                 }
                 else if (element.type == "lipsync") {
                     if(this.scrubberPosition+1 >= element.ending_offset){

app/pages/PageEnigma/reducers/appUiReducer/values.tsx

@@ -4,11 +4,12 @@ export const initialStateValues : State = {
   timelineHeight: 260,
   viewMode: VIEW_MODES.EDITOR,
   showEditorLoader: {
-    isShowing:true,
-    message: 'loading Editor Engine 🦊'
+    isShowing:false,
+    message: 'Loading Editor Engine 🦊'
   },
   showEditorLoadingBar: {
-    isShowing:false,
-    progress:0,
+    isShowing:true,
+    progress:5,
+    message: 'Loading Editor Engine 🦊'
   }
 }

app/routes/ComponentLibrary.tsx

@@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import { 
   Button,
   ButtonLink,
-  H4, 
+  H1,H2,H3,H4,H5,H6,P,Label,
   LoadingBar,
   LoadingDotsTyping,
   LoadingDotsBricks,
@@ -35,65 +35,84 @@ export default function ComponentLibrary () {
   },[progress]);
 
   return(
-    <div className='bg-ui-panel w-10/12 max-w-7xl h-full min-h-96 mx-auto my-6 rounded-lg p-6'>
+    <div
+      className="fixed w-full overflow-scroll"
+      style={{height: "calc(100% - 72px)"}}
+    >
+      <div
+        className='bg-ui-panel w-10/12 max-w-7xl mx-auto my-6 rounded-lg p-6'
+      >
 
-      <div className="flex flex-col gap-2 mb-4">
-        <H4>Buttons</H4>
-        <div className="flex gap-2">
-          <Button>Primary</Button>
-          <Button variant="secondary">Secondary</Button>
-          <Button className="bg-brand-tertiary hover:bg-brand-teriary-400 focus-visible:outline-brand-tertiary">
-            Prelim Tertiary
-          </Button>
-          <Button disabled>Disabled</Button>
-        </div>
-      </div>
-
-      <div className="flex flex-col gap-2 mb-4">
-        <H4>ButtonLink</H4>
-        <ButtonLink to="/">Back to /</ButtonLink>
-      </div>
-
-      <div className="flex flex-col gap-2 mb-4">
-        <H4>Loading Dots</H4>
-        <div className="flex gap-2">
-          <div className="w-60 h-40 rounded-lg overflow-hidden">
-            <LoadingDotsTyping />
-          </div>
-          <div className="w-60 h-40 rounded-lg overflow-hidden">
-            <LoadingDotsBricks />
+        <div className="flex flex-col gap-2 mb-4">
+          <H1>Typography</H1>
+          <H1>H1. This is an H1 Heading</H1>
+          <H2>H2. This is an H2 Heading</H2>
+          <H3>H3. This is an H3 Heading</H3>
+          <H4>H4. This is an H4 Heading</H4>
+          <Label>Label. Is essentially same as H4</Label>
+          <P>P. Is an normal Paragraph</P>
+          <H5>H5. This is an H5 Heading</H5>
+          <H6>H6. This is an H6 Heading</H6>
+    
+          <hr className="mt-2 mb-4"/>
+          <H1>Components</H1>
+          <H4>Buttons</H4>
+          <div className="flex gap-2">
+            <Button>Primary</Button>
+            <Button variant="secondary">Secondary</Button>
+            <Button className="bg-brand-tertiary hover:bg-brand-teriary-400 focus-visible:outline-brand-tertiary">
+              Prelim Tertiary
+            </Button>
+            <Button disabled>Disabled</Button>
           </div>
         </div>
-      </div>
 
-      <div className="flex flex-col gap-2 mb-4">
-        <H4>Loading Bar</H4>
-        <div className="flex gap-2 items-center">
+        <div className="flex flex-col gap-2 mb-4">
+          <H4>ButtonLink</H4>
+          <ButtonLink to="/">Back to /</ButtonLink>
+        </div>
+
+        <div className="flex flex-col gap-2 mb-4">
+          <H4>Loading Dots</H4>
+          <div className="flex gap-2">
+            <div className="w-60 h-40 rounded-lg overflow-hidden">
+              <LoadingDotsTyping />
+            </div>
+            <div className="w-60 h-40 rounded-lg overflow-hidden">
+              <LoadingDotsBricks />
+            </div>
+          </div>
+        </div>
+
+        <div className="flex flex-col gap-2 mb-4">
+          <H4>Loading Bar</H4>
+          <div className="flex gap-2 items-center">
+            <LoadingBar
+              progress={progress}
+              isShowing={progress !== 100}
+              wrapperClassName="rounded-lg"
+              message="this takes progress from parent"
+            />
+            <Button
+              className="h-fit"
+              onClick={()=>setProgress(0)}
+            >
+              Reset
+            </Button>
+          </div>
           <LoadingBar
-            progress={progress}
-            isShowing={progress !== 100}
-            wrapperClassName="rounded-lg"
-            message="this takes progress from parent"
+            label="labeling the bar"
+            message="displaying a message"
+          />
+          <LoadingBar
+            label="useFakeTimer = 30000 (30secs)"
+            useFakeTimer={30000}
+          />
+          <LoadingBar
+            label="useFakeTimer = 3000 (3secs)"
+            useFakeTimer={3000}
           />
-          <Button
-            className="h-fit"
-            onClick={()=>setProgress(0)}
-          >
-            Reset
-          </Button>
         </div>
-        <LoadingBar
-          label="labeling the bar"
-          message="displaying a message"
-        />
-        <LoadingBar
-          label="useFakeTimer = 30000 (30secs)"
-          useFakeTimer={30000}
-        />
-        <LoadingBar
-          label="useFakeTimer = 3000 (3secs)"
-          useFakeTimer={3000}
-        />
       </div>
     </div>
   );