Merge pull request #55 from storytold/feature/resume-single-frame

Feature/resume single frame

app/pages/PageEnigma/comps/PreviewEngineCamera/index.tsx

@@ -23,11 +23,7 @@ export const PreviewEngineCamera = ()=>{
         </div>
         <div className="relative rounded-lg border border-white box overflow-hidden">
           
-          <img
-            //shoot that datat from engine context to this image
-            className="aspect-video max-h-40" 
-            src="/resources/uiAssets/video_player_placeholder.gif"
-          />
+          <canvas className="aspect-video max-h-40" id="camera-view"></canvas>
           <div className="absolute w-full h-full top-0 left-0">
             <LoadingDotsTyping isShowing={showLoader}/>
           </div>

app/pages/PageEnigma/comps/ViewSideBySide/index.tsx

@@ -1,11 +1,15 @@
-import { useEffect, useState, } from "react";
+import { useEffect, useState, useContext } from "react";
 
 import { LoadingDotsBricks } from "~/components";
+import { EngineContext } from "../../contexts/EngineContext";
 
 export const ViewSideBySide = ()=>{
   const [showLoader, setShowLoader] = useState(true);
+  const editorEngine = useContext(EngineContext);
+
   useEffect(()=>{
-    setTimeout(()=>setShowLoader(false), 1000);
+    editorEngine?.generateFrame();
+    setTimeout(()=>setShowLoader(false), 200);
   },[ ]);
 
   return (
@@ -18,11 +22,7 @@ export const ViewSideBySide = ()=>{
           <label>Raw Preview</label>
         </div>
         <div className="relative">
-          <img
-            //shoot that datat from engine context to this image
-            className="aspect-video max-h-150" 
-            src="/resources/uiAssets/video_player_placeholder.gif"
-          />
+        <img className="aspect-video max-h-150 border" id="raw-preview"></img>
         </div>
       </div>
 
@@ -31,11 +31,7 @@ export const ViewSideBySide = ()=>{
           <label>Stylized Preview</label>
         </div>
         <div className="relative">
-          <img
-            //shoot that datat from engine context to this image
-            className="aspect-video max-h-150" 
-            src="/resources/uiAssets/video_player_placeholder.gif"
-          />
+        <img className="aspect-video max-h-150 border" id="stylized-preview"></img>
         </div>
       </div>
       <div className="absolute top-0 left-0 w-full h-full">

app/pages/PageEnigma/datastructures/clips/lipsync_clip.ts

@@ -113,6 +113,11 @@ step() {
 }
 
 reset() {
+  if (this.lipsync.face != undefined) {
+    this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ee] = 0;
+    this.lipsync.face.morphTargetInfluences[this.blendshape_helper.ah] = 0
+    this.lipsync.face.morphTargetInfluences[this.blendshape_helper.oh] = 0;
+  }
   this.lipsync = new LipSync();
 }
 

app/pages/PageEnigma/js/editor.ts

@@ -45,8 +45,10 @@ class Editor {
   camera: any;
   render_camera: any;
   renderer: THREE.WebGLRenderer | undefined;
+  rawRenderer: THREE.WebGLRenderer | undefined;
   clock: THREE.Clock | undefined;
   canvReference: any;
+  canvasRenderCamReference: any;
   composer: EffectComposer | undefined;
   effectFXAA: EffectComposer | undefined;
   outlinePass: OutlinePass | undefined;
@@ -86,6 +88,7 @@ class Editor {
   lockControls: PointerLockControls | undefined;
   cam_obj: THREE.Object3D | undefined;
   renderPass: RenderPass | undefined;
+  generating_preview: boolean;
 
   camera_person_mode: boolean;
   current_scene_media_token: string | null;
@@ -120,9 +123,11 @@ class Editor {
     // Clock, scene and camera essentials.
     this.activeScene = new Scene("" + this.version);
     this.activeScene.initialize();
+    this.generating_preview = false;
     this.camera;
     this.render_camera;
     this.renderer;
+    this.rawRenderer;
     this.clock;
     this.canvReference = null;
     this.cam_obj;
@@ -168,6 +173,8 @@ class Editor {
     this.render_width = 1280;
     this.render_height = 720;
 
+    this.canvasRenderCamReference;
+
     this.audio_engine = new AudioEngine();
     this.transform_engine = new TransformEngine(this.version);
     this.lipsync_engine = new LipSyncEngine();
@@ -215,16 +222,18 @@ class Editor {
 
     // Gets the canvas.
     this.canvReference = document.getElementById("video-scene");
+    this.canvasRenderCamReference = document.getElementById("camera-view");
+
     // Base width and height.
     let width = this.canvReference.width;
     let height = this.canvReference.height;
     // Sets up camera and base position.
-    this.camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50);
+    this.camera = new THREE.PerspectiveCamera(70, width / height, 0.15, 30);
     this.camera.position.z = 3;
     this.camera.position.y = 3;
     this.camera.position.x = -3;
 
-    this.render_camera = new THREE.PerspectiveCamera(70, width / height, 0.1, 50);
+    this.render_camera = new THREE.PerspectiveCamera(70, width / height, 0.15, 30);
 
     // Base WebGL render and clock for delta time.
     this.renderer = new THREE.WebGLRenderer({
@@ -233,6 +242,12 @@ class Editor {
       preserveDrawingBuffer: true,
     });
 
+    this.rawRenderer = new THREE.WebGLRenderer({
+      antialias: false,
+      canvas: this.canvasRenderCamReference,
+      preserveDrawingBuffer: true,
+    });
+
     this.renderer.shadowMap.enabled = true;
     this.clock = new THREE.Clock();
     // Resizes the renderer.
@@ -281,11 +296,6 @@ class Editor {
     this.renderer.domElement.addEventListener("mouseup", this.onMouseUp.bind(this), false);
     this.renderer.domElement.addEventListener("onContextMenu", this.onContextMenu.bind(this), false);
 
-    this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
-    if (this.cam_obj) {
-      this.addTransformClipBase("Camera Object", "camera", this.cam_obj, 0, 150)
-    }
-
     // saving state of the scene
     this.current_scene_media_token = null;
     this.current_scene_glb_media_token = null;
@@ -311,6 +321,8 @@ class Editor {
       this.addTransformClipBase("Camera Object", "camera", this.cam_obj, 0, 150);
     }
 
+    console.log(this.cam_obj);
+
     this.dispatchAppUiState({
       type: APPUI_ACTION_TYPES.UPDATE_EDITOR_LOADINGBAR,
       payload: {
@@ -505,6 +517,7 @@ class Editor {
 
   switchCameraView() {
     this.camera_person_mode = !this.camera_person_mode;
+    console.log("camera")
     if (this.cam_obj) {
       if (this.camera_person_mode) {
         this.last_cam_pos.copy(this.camera.position);
@@ -521,11 +534,12 @@ class Editor {
         if (this.cameraViewControls) {
           this.cameraViewControls.enabled = true;
         }
-
+        this.cam_obj.scale.set(0,0,0);
         if (this.activeScene.hot_items) {
           this.activeScene.hot_items.forEach((element) => {
             element.visible = false;
           });
+        
         }
       } else {
         this.camera.position.copy(this.last_cam_pos);
@@ -539,6 +553,7 @@ class Editor {
         if (this.cameraViewControls) {
           this.cameraViewControls.enabled = false;
         }
+        this.cam_obj.scale.set(1,1,1);
         if (this.activeScene.hot_items) {
           this.activeScene.hot_items.forEach((element) => {
             element.visible = true;
@@ -745,6 +760,14 @@ class Editor {
 
   // Basicly Unity 3D's update loop.
   updateLoop(time: number) {
+
+    if (this.cam_obj == undefined) {
+      this.cam_obj = this.activeScene.get_object_by_name("::CAM::");
+      if (this.cam_obj) {
+        this.addTransformClipBase("Camera Object", "camera", this.cam_obj, 0, 150)
+      }
+    }
+
     // Updates debug stats.
     if (this.stats != null) {
       this.stats.update();
@@ -907,7 +930,9 @@ class Editor {
   }
 
   async generateFrame() {
-    if (this.renderer) {
+    if (this.renderer && !this.generating_preview) {
+      this.removeTransformControls();
+      this.generating_preview = true;
       this.activeScene.renderMode(true);
       if (this.activeScene.hot_items) {
         this.activeScene.hot_items.forEach(element => {
@@ -925,32 +950,46 @@ class Editor {
       let imgData = this.renderer.domElement.toDataURL();
       this.activeScene.renderMode(false);
       this.onWindowResize();
-      let ffmpeg = createFFmpeg({ log: false });
-      await ffmpeg.load();
-      await ffmpeg.FS(
-        "writeFile",
-        `render.png`,
-        await fetchFile(imgData),
-      );
-      await ffmpeg.run('-i', `render.png`, 'render.mp4');
-      let output = await ffmpeg.FS("readFile", 'render.mp4');
-      const blob = new Blob([output.buffer], { type: "video/mp4" });
 
-      //const blob = new Blob([imgData], { type: "image/png" });
-      let url = await this.api_manager.uploadMediaFrameGeneration(blob, "render.mp4", "anime_ghibli", "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.", "");
+      const rawPreview: HTMLVideoElement | null = document.getElementById("raw-preview") as HTMLVideoElement;
+      if (rawPreview) {
+        rawPreview.src = imgData;
 
-      const downloadLink = document.createElement("a");
-      downloadLink.href = url;
-      downloadLink.download = "render.mp4";
-      document.body.appendChild(downloadLink);
-      downloadLink.click();
-      // Clean up
-      URL.revokeObjectURL(url);
-      document.body.removeChild(downloadLink);
-      
-      return new Promise((resolve, reject) => {
-        resolve(url);
-      });
+
+        let ffmpeg = createFFmpeg({ log: false });
+        await ffmpeg.load();
+        await ffmpeg.FS(
+          "writeFile",
+          `render.png`,
+          await fetchFile(imgData),
+        );
+        await ffmpeg.run('-i', `render.png`, 'render.mp4');
+        let output = await ffmpeg.FS("readFile", 'render.mp4');
+        const blob = new Blob([output.buffer], { type: "video/mp4" });
+
+        let url = await this.api_manager.uploadMediaFrameGeneration(blob, "render.mp4", "anime_ghibli", "((masterpiece, best quality, 8K, detailed)), colorful, epic, fantasy, (fox, red fox:1.2), no humans, 1other, ((koi pond)), outdoors, pond, rocks, stones, koi fish, ((watercolor))), lilypad, fish swimming around.", "");
+        console.log(url);
+
+        const stylePreview: HTMLVideoElement | null = document.getElementById("stylized-preview") as HTMLVideoElement;
+        if (stylePreview) {
+          stylePreview.src = url;
+          stylePreview.width = rawPreview.width;
+          stylePreview.height = rawPreview.height;
+        }
+        else {
+          console.log("No style preview window.")
+        }
+
+        this.generating_preview = false;
+
+        return new Promise((resolve, reject) => {
+          resolve(url);
+        });
+      }
+      else {
+        console.log("No raw preview window.")
+      }
+      this.generating_preview = false;
     }
   }
 
@@ -999,8 +1038,9 @@ class Editor {
 
   // Render the scene to the camera.
   renderScene() {
-    if (this.composer != null && !this.rendering) {
+    if (this.composer != null && !this.rendering && this.rawRenderer) {
       this.composer.render();
+      this.rawRenderer.render(this.activeScene.scene, this.render_camera);
     } else if (this.renderer && this.render_camera) {
       this.renderer.setSize(this.render_width, this.render_height);
       this.renderer.render(this.activeScene.scene, this.render_camera);

app/pages/PageEnigma/js/scene.ts

@@ -64,7 +64,7 @@ class Scene {
         let obj = new THREE.Mesh(geometry, material);
         obj.position.copy(pos);
         obj.receiveShadow = visable;
-        if(this.hot_items != undefined){
+        if (this.hot_items != undefined) {
             this.hot_items.push(obj);
         }
         this.scene.add(obj);
@@ -75,15 +75,18 @@ class Scene {
     }
 
     _create_camera_obj() {
-        let cam_obj: THREE.Object3D = new THREE.Object3D();
-        cam_obj.userData["name"] = "::CAM::";
-        cam_obj.name = "::CAM::";
-        this.scene.add(cam_obj);
+        this.load_glb("/resources/models/camera/camera.glb", false).then((cam_obj) => {
+            cam_obj.userData["name"] = "::CAM::";
+            cam_obj.name = "::CAM::";
+            cam_obj.position.set(0, 0.6, 1.5);
+            this.scene.add(cam_obj);
+        });
     }
 
-    renderMode(enabled:boolean=true) {
-        if(this.gridHelper == undefined) { return; }
-        if(enabled) {
+
+    renderMode(enabled: boolean = true) {
+        if (this.gridHelper == undefined) { return; }
+        if (enabled) {
             this._disable_skybox();
             this.scene.remove(this.gridHelper);
         } else {
@@ -92,7 +95,7 @@ class Scene {
         }
     }
 
-    async load_glb(filepath: string) { //: Promise<THREE.Object3D> {
+    async load_glb(filepath: string, auto_add: boolean = true): Promise<THREE.Object3D> { //: Promise<THREE.Object3D> {
         return new Promise((resolve) => {
             let glbLoader = new GLTFLoader();
             glbLoader.load(filepath, (glb) => {
@@ -106,7 +109,12 @@ class Scene {
                             c.material.transparent = false;
                         }
                     });
-                    this.scene.add(child);
+                    if (child.type == "Group") {
+                        if (auto_add) { this.scene.add(child.children[0]); }
+                        resolve(child.children[0]);
+                        return;
+                    }
+                    if (auto_add) { this.scene.add(child); }
                     resolve(child);
                 });
             });

app/pages/PageEnigma/js/timeline.ts

@@ -130,6 +130,7 @@ export class TimeLine {
                 this.audio_engine.loadClip(element.media_id);
             }
             else if (element.type == "animation") {
+                this.animation_engine.clips[element.object_uuid].stop();
             }
             else if (element.type == "lipsync") {
                 this.lipSync_engine.clips[element.object_uuid].reset();
@@ -199,6 +200,10 @@ export class TimeLine {
                 this.stop()
             }
         }
+
+        if (this.scrubber_frame_position >= this.timeline_limit) {
+            await this.resetScene();
+        }
     }
 
     private async stop(): Promise<void> {