Merge pull request #38 from storytold/feature/camera-mode

Feature/camera mode

app/modules/SidePanel/SidePanel.tsx

@@ -27,7 +27,6 @@ export const SidePanel = ({ children }: Props) => {
       <div className="relative w-[23.5rem] h-full">
         {children}
       </div>
-
     </div>
   );
 };

app/pages/PageEnigma/PageEnigmaComponent.tsx

@@ -99,4 +99,4 @@ export const PageEnigmaComponent = () => {
       </AppUIProvider>
     </div>
   );
-};
+};

app/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx

@@ -27,7 +27,7 @@ export const ControlsTopButtons = () => {
   };
 
   const handleButtonPlayBack = () => {
-    editorEngine?.start_playback();
+    editorEngine?.startPlayback();
   };
 
   const handleButtonTest = () => {
@@ -71,13 +71,17 @@ export const ControlsTopButtons = () => {
     }
   };
 
+  //const handleButtonRender = () => {
+  //  editorEngine?.take_timeline_cam_clip();
+  //};
+
   const handleButtonRender = () => {
-    editorEngine?.take_timeline_cam_clip();
-  };
-  const handleButtonPlay = () => {
-    editorEngine?.start_playback();
+    editorEngine?.generateVideo();
   };
 
+  const handleButtonTakeFrame = () => {
+    editorEngine?.take_timeline_cam_clip();
+  };
 
   // const handleButtonCameraView = () => {
   //   editorEngine?.change_camera_view();
@@ -146,9 +150,9 @@ export const ControlsTopButtons = () => {
       </div>
       <div className="flex gap-2">
 
-        <Button onClick={handleButtonLoad}>Load</Button>
-        <Button onClick={handleButtonRender}>Take Frame</Button>
-        <Button onClick={handleButtonPlay}>Play</Button>
+        <Button onClick={handleButtonLoad}>Test Button</Button>
+        <Button onClick={handleButtonTakeFrame}>Take Frame</Button>
+        <Button onClick={handleButtonRender}>Render</Button>
         {/* <Button onClick={handleButtonLoad}>Load</Button>
         <Button onClick={handleButtonRender}>Render</Button> */}
         <Button

app/pages/PageEnigma/comps/ControlsVideo/ControlsVideo.tsx

@@ -1,3 +1,5 @@
+import { useContext } from "react";
+
 import {
   faBackwardFast,
   faBackwardStep,
@@ -5,9 +7,12 @@ import {
   faForwardStep,
   faPlay,
 } from "@fortawesome/pro-solid-svg-icons";
+import { EngineContext } from "../../contexts/EngineContext";
 import { ButtonIcon } from "~/components";
 
 export const ControlsVideo = () => {
+  const editorEngine = useContext(EngineContext);
+  
   const handleBackwardFast = () => {
     console.log("Controls Video: Backward-Fast clicked");
   };
@@ -16,6 +21,8 @@ export const ControlsVideo = () => {
   };
   const handlePlay = () => {
     console.log("Controls Video: Play clicked");
+    editorEngine?.startPlayback();
+    
   };
   const handleForwardStep = () => {
     console.log("Controls Video: Forward-Step clicked");

app/pages/PageEnigma/comps/Helmet.tsx

@@ -0,0 +1,24 @@
+import { useContext, useEffect, useState } from "react";
+import { faSparkles } from "@fortawesome/pro-solid-svg-icons";
+import { TopBarHelmet } from "~/modules/TopBarHelmet/TopBarHelmet";
+import { Button } from "~/components";
+import { EngineContext } from "../contexts/EngineContext";
+
+export const Helmet = () => {
+  const [helmetKey, setHelmetKey] = useState<number>(Date.now());
+  const engineContext = useContext(EngineContext);
+  
+  const handleGenerateVideo = () => {
+    engineContext?.generateVideo();
+  };
+  
+  useEffect(()=>{
+    setHelmetKey(Date.now());
+  },[engineContext]);
+
+  return (
+    <TopBarHelmet key={helmetKey}>
+      <Button icon={faSparkles} onClick={handleGenerateVideo}>Generate Movie</Button>
+    </TopBarHelmet>
+  );
+}

app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx

@@ -8,7 +8,7 @@ export const PreviewEngineCamera = ()=>{
 
   const [showLoader, setShowLoader] = useState<boolean>(true);
   useEffect(()=>{
-    setTimeout(()=>setShowLoader(false), 5000);
+    setTimeout(()=>setShowLoader(false), 500);
   },[]);
   return (
     <div

app/pages/PageEnigma/datastructures/clips/animation_clip.ts

@@ -74,19 +74,16 @@ export class AnimationClip  {
           this.clip_action.play();
         }
       }
-      console.log("Play")
     }
   }
 
   step(deltatime: number) {
     if (this.mixer == null) { return; }
     this.mixer?.update(deltatime);
-    console.log("Update")
   }
 
   stop() {
     this.mixer?.stopAllAction();
-    console.log("stop.")
   }
 
   toJSON(): string {

app/pages/PageEnigma/js/api_manager.ts

@@ -162,7 +162,7 @@ class APIManager {
    * @param media_file_token
    * @returns
    */
-  private async getMediaFile(media_file_token: string): Promise<string> {
+  public async getMediaFile(media_file_token: string): Promise<string> {
     let api_base_url = "https://api.fakeyou.com";
     let url = `${api_base_url}/v1/media_files/file/${media_file_token}`;
     let response = await fetch(url);

app/pages/PageEnigma/js/editor.ts

@@ -12,7 +12,7 @@ import { SMAAPass } from "three/addons/postprocessing/SMAAPass.js";
 import { SAOPass } from "three/addons/postprocessing/SAOPass.js";
 import { UnrealBloomPass } from "three/addons/postprocessing/UnrealBloomPass.js";
 import { BokehPass } from "three/addons/postprocessing/BokehPass.js";
-import { createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg";
+import { FFmpeg, createFFmpeg, fetchFile } from "@ffmpeg/ffmpeg";
 import AudioEngine from "./audio_engine.js";
 import TransformEngine from "./transform_engine.js";
 import { TimeLine, TimelineDataState } from "./timeline.js";
@@ -43,6 +43,7 @@ class Editor {
   version: number;
   activeScene: Scene;
   camera: any;
+  render_camera: any;
   renderer: THREE.WebGLRenderer | undefined;
   clock: THREE.Clock | undefined;
   canvReference: any;
@@ -86,12 +87,14 @@ class Editor {
   cam_obj: THREE.Object3D | undefined;
   renderPass: RenderPass | undefined;
 
+  camera_person_mode: boolean;
   current_scene_media_token: string | null;
   current_scene_glb_media_token: string | null;
 
   can_initialize: boolean;
   dispatchAppUiState: any; // todo figure out the type
-  camera_person_mode: boolean;
+  render_width: number;
+  render_height: number;
 
   // Default params.
   constructor() {
@@ -115,6 +118,7 @@ class Editor {
     this.activeScene = new Scene("" + this.version);
     this.activeScene.initialize();
     this.camera;
+    this.render_camera;
     this.renderer;
     this.clock;
     this.canvReference = null;
@@ -158,6 +162,9 @@ class Editor {
     this.max_length = 10;
     // Audio Engine Test.
 
+    this.render_width = 1280;
+    this.render_height = 720;
+
     this.audio_engine = new AudioEngine();
     this.transform_engine = new TransformEngine(this.version);
     this.lipsync_engine = new LipSyncEngine();
@@ -206,6 +213,9 @@ class Editor {
     this.camera.position.z = 3;
     this.camera.position.y = 3;
     this.camera.position.x = -3;
+
+    this.render_camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50);
+
     // Base WebGL render and clock for delta time.
     this.renderer = new THREE.WebGLRenderer({
       antialias: true,
@@ -247,7 +257,7 @@ class Editor {
     window.addEventListener("mousemove", this.onMouseMove.bind(this), false);
     window.addEventListener("click", this.onMouseClick.bind(this), false);
     // Base control and debug stuff remove debug in prod.
-    this._initialize_control();
+    this._initializeControl();
     // Resets canvas size.
     this.onWindowResize();
     // Creates the main update loop.
@@ -257,6 +267,16 @@ class Editor {
 
     this._test_demo();
 
+
+    this.renderer.domElement.addEventListener("mousedown", this.onMouseDown.bind(this), false);
+    this.renderer.domElement.addEventListener("mouseup", this.onMouseUp.bind(this), false);
+    this.renderer.domElement.addEventListener("onContextMenu", this.onContextMenu.bind(this), false);
+
+    this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
+    if (this.cam_obj) {
+      this.add_transform_clip_base("Camera Object", this.cam_obj, 0, 150)
+    }
+
     // saving state of the scene
     this.current_scene_media_token = null;
     this.current_scene_glb_media_token = null;
@@ -322,6 +342,7 @@ class Editor {
 
     const loaded_scene = load_scene_state_response.data["scene"];
 
+    if (load_scene_state_response.data == null) { return; }
     // Load these so you can rewrite the scene glb using it's token.
     this.current_scene_media_token =
       load_scene_state_response.data["scene_media_file_token"];
@@ -573,6 +594,18 @@ class Editor {
         400,
       ),
     );
+
+
+    this.audio_engine.loadClip("m_h33vytxs5eqqqf07nsy14qzrf9ww4v");
+
+    await this.timeline.addPlayableClip(new ClipUI(
+      1.0,
+      "audio",
+      "AudioClip",
+      "m_h33vytxs5eqqqf07nsy14qzrf9ww4v",
+      "",
+      0,
+      50));
   }
 
   // Configure post processing.
@@ -646,6 +679,7 @@ class Editor {
   render_mode() {
     this.rendering = !this.rendering;
     this.activeScene.render_mode(this.rendering);
+    console.log("works.")
 
     //if (this.rendering) {
     //    this._remove_post_processing()
@@ -731,19 +765,14 @@ class Editor {
       }
     }
 
+    if (this.render_camera && this.cam_obj) {
+      this.render_camera.position.copy(this.cam_obj.position);
+      this.render_camera.rotation.copy(this.cam_obj.rotation);
+    }
+
     this.timeline.update(delta_time);
 
-    this.render_scene();
-    if (this.capturer != null) {
-      this.capturer.capture(this.renderer.domElement);
-    } // Record scene.
-  }
-
-  start_playback() {
-    this.timeline.isPlaying = true;
-    if (!this.camera_person_mode) {
-      this.switchCameraView();
-    }
+    this.renderScene();
   }
 
   change_mode(type: any) {
@@ -777,6 +806,56 @@ class Editor {
     return blob;
   }
 
+  async convertAudioClip(itteration: number, ffmpeg: FFmpeg, clip: ClipUI) {
+    let video_og = itteration + 'tmp.mp4';
+    let wav_name = itteration + 'tmp.wav';
+    let new_video = (itteration + 1) + 'tmp.mp4';
+    let startFrame = clip.start_offset;
+    let endFrame = clip.ending_offset;
+
+
+    if (endFrame > this.timeline.timeLineLimit) {
+      endFrame = this.timeline.timeLineLimit;
+    }
+    if (startFrame > this.timeline.timeLineLimit) {
+      startFrame = this.timeline.timeLineLimit-1;
+    }
+
+    const startTime = startFrame / this.cap_fps;
+    const endTime = endFrame / this.cap_fps;
+    let end = endTime - startTime;
+
+    let duration = this.timeline.timeLineLimit / this.cap_fps;
+
+    let audioSegment = "as_" + wav_name;
+    await ffmpeg.FS('writeFile', wav_name, await fetchFile(await this.api_manager.getMediaFile(clip.media_id)));
+    await ffmpeg.run('-i',
+      wav_name,
+      '-ss', '0',
+      '-to', '' + end,
+      "-max_muxing_queue_size", "999999",
+      audioSegment);
+
+    await ffmpeg.run('-i', video_og,"-max_muxing_queue_size", "999999", `${itteration}empty_tmp.wav`);
+
+    await ffmpeg.run(
+      '-i', `${itteration}empty_tmp.wav`, 
+      '-i', audioSegment, 
+      '-filter_complex', "[1:a]adelay="+startTime*1000+"|"+startTime*1000+"[a1];[0:a][a1]amix=inputs=2[a]",
+      "-map", "[a]",
+      `${itteration}final_tmp.wav`)
+
+    await ffmpeg.run(
+      '-i', video_og, 
+      '-i', `${itteration}final_tmp.wav`, 
+      '-c:v', 'copy',
+      '-c:a', 'aac',
+      '-map', '0:v:0',
+      '-map', '1:a:0',
+      '-strict', 'experimental',
+      new_video)
+  }
+
   async stopPlayback() {
     this.render_mode();
     let ffmpeg = createFFmpeg({ log: true });
@@ -794,35 +873,78 @@ class Editor {
       "" + this.cap_fps,
       "-i",
       "image%d.png",
-      "output.mp4",
+      "-f",
+      "lavfi",
+      "-i",
+      "anullsrc", // This adds a silent audio track
+      "-max_muxing_queue_size",
+      "999999",
+      "-c:v",
+      "libx264", // Specify video codec (optional, but recommended for MP4)
+      "-c:a",
+      "aac", // Specify audio codec (optional, but recommended for MP4)
+      "-shortest", // Ensure output duration matches the shortest stream (video or audio)
+      "0tmp.mp4"
     );
-    let output = await ffmpeg.FS("readFile", "output.mp4");
+
+
+    let itteration = 0;
+
+    for (const clip of this.timeline.timelineItems) {
+      if (clip.type == "lipsync" || clip.type == "audio") {
+        await this.convertAudioClip(itteration, ffmpeg, clip);
+        itteration += 1;
+      }
+    };
+
+    let output = await ffmpeg.FS("readFile", itteration + "tmp.mp4");
     // Create a Blob from the output file for downloading
     const blob = new Blob([output.buffer], { type: "video/mp4" });
     const url = URL.createObjectURL(blob);
-    await this.api_manager.uploadMedia(blob, "output.mp4");
-    // Create a link to download the file
     const downloadLink = document.createElement("a");
     downloadLink.href = url;
-    downloadLink.download = "output.mp4";
+    downloadLink.download = "render.mp4";
     document.body.appendChild(downloadLink);
     downloadLink.click();
     // Clean up
     URL.revokeObjectURL(url);
     document.body.removeChild(downloadLink);
+  
+  
+    let data = await this.api_manager.uploadMedia(blob, "tmp.wav");
+    console.log(data);
+    // Create a link to download the file
+
+  }
+
+  generateVideo() {
+    console.log("Generating video...");
+    if (this.rendering) { return; }
+    this.startPlayback();
+    this._initializeRecording();
+    this.rendering = true;
+    this.activeScene.render_mode(this.rendering);
+    if (this.activeScene.hot_items) {
+      this.activeScene.hot_items.forEach(element => {
+        element.visible = false;
+      });
+    }
   }
 
   startPlayback() {
-    this.playback_location = 0;
-    this._initialize_recording();
+    this.timeline.isPlaying = true;
+    this.timeline.scrubberPosition = 0;
+    if (!this.camera_person_mode) {
+      this.switchCameraView();
+    }
   }
 
   // Initializes transform x y z changes.
-  _initialize_control() {
+  _initializeControl() {
     if (this.control == undefined) {
       return;
     }
-    this.control.addEventListener("change", this.render_scene.bind(this));
+    this.control.addEventListener("change", this.renderScene.bind(this));
     this.control.addEventListener("dragging-changed", (event: any) => {
       if (this.orbitControls == undefined) {
         return;
@@ -836,18 +958,18 @@ class Editor {
     this.activeScene.scene.add(this.control);
   }
 
-  // Initializes CCapture for capturing the scene to send over to backend.
-  _initialize_recording() {
+  _initializeRecording() {
     this.frame_buffer = [];
     this.render_timer = 0;
   }
 
   // Render the scene to the camera.
-  render_scene() {
-    if (this.composer != null) {
+  renderScene() {
+    if (this.composer != null && !this.rendering) {
       this.composer.render();
-    } else if (this.renderer && this.camera) {
-      this.renderer.render(this.activeScene.scene, this.camera);
+    } else if (this.renderer && this.render_camera) {
+      this.renderer.setSize(this.render_width, this.render_height);
+      this.renderer.render(this.activeScene.scene, this.render_camera);
     } else {
       console.error("Could not render to canvas no render or composer!");
     }
@@ -857,10 +979,11 @@ class Editor {
       let imgData = this.renderer.domElement.toDataURL();
       this.frame_buffer.push(imgData);
       this.render_timer += this.clock.getDelta();
-      if (this.playback_location >= this.fps_number * 3) {
+      if (this.timeline.isPlaying == false) {
         this.stopPlayback();
         this.playback_location = 0;
         this.rendering = false;
+        this.onWindowResize();
       }
     }
   }
@@ -883,6 +1006,12 @@ class Editor {
     if (this.composer != null) {
       this.composer.setSize(width, height);
     }
+
+    if (this.render_camera == undefined) { return; }
+
+    //this.renderer.setSize(this.render_width, this.render_height);
+    this.render_camera.aspect = this.render_width / this.render_height;
+    this.render_camera.updateProjectionMatrix();
   }
 
   onContextMenu(event: any) {

app/pages/PageEnigma/js/timeline.ts

@@ -72,7 +72,7 @@ export class TimeLine {
         scene: Scene) {
         
         this.timelineItems = []
-        this.timeLineLimit = 60 * 10 // 10 seconds
+        this.timeLineLimit = 60 * 5 // 5 seconds
      
         this.isPlaying = false
         this.scrubberPosition = 0 // in frames into the tl
@@ -151,7 +151,7 @@ export class TimeLine {
             else if (element.type == "animation") {
             }
             else if (element.type == "lipsync") {
-                
+                this.lipSyncEngine.clips[element.object_uuid].reset();
             }
             else {
                 this.stop()
@@ -184,8 +184,12 @@ export class TimeLine {
                     }
                 }
                 else if (element.type == "audio") {
-                    // global audio track
-                    this.audioEngine.playClip(element.media_id)
+                    if(this.scrubberPosition+1 >= element.ending_offset){
+                        this.audioEngine.stopClip(element.media_id);
+                    }
+                    else {
+                        this.audioEngine.playClip(element.media_id);
+                    }
                 }
                 else if (element.type == "lipsync") {
                     if(this.scrubberPosition+1 >= element.ending_offset){