Make 3D state save and load on tab change

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/SceneMetadata.ts

@@ -0,0 +1,32 @@
+import { getArtStyle } from "~/enums/ArtStyle";
+import {
+  cameraAspectRatio,
+  cinematic,
+  enginePreProcessing,
+  faceDetail,
+  globalIPAMediaToken,
+  lipSync,
+  styleStrength,
+  upscale,
+} from "~/pages/PageEnigma/signals";
+import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
+import Editor from "./editor";
+
+export const getSceneGenerationMetaData = (
+  editorEngine: Editor,
+): SceneGenerationMetaData => {
+  // when this is called, editor engine is guarunteed by it's caller
+  return {
+    positivePrompt: editorEngine.positive_prompt,
+    negativePrompt: editorEngine.negative_prompt,
+    artisticStyle: getArtStyle(editorEngine.art_style.toString()),
+    cameraAspectRatio: cameraAspectRatio.value,
+    globalIPAMediaToken: globalIPAMediaToken.value || undefined,
+    upscale: upscale.value,
+    faceDetail: faceDetail.value,
+    styleStrength: styleStrength.value,
+    lipSync: lipSync.value,
+    cinematic: cinematic.value,
+    enginePreProcessing: enginePreProcessing.value,
+  };
+};

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts

@@ -50,24 +50,23 @@ import { SceneManager } from "./scene_manager_api";
 import { CustomOutlinePass } from "./CustomOutlinePass.js";
 import FindSurfaces from "./FindSurfaces.js";
 
-
 import Stats from "three/examples/jsm/libs/stats.module.js";
 import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine";
 import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera";
 
-
 export type EditorInitializeConfig = {
   sceneToken: string;
   editorCanvasEl: HTMLCanvasElement;
   camViewCanvasEl: HTMLCanvasElement;
   sceneContainerEl: HTMLDivElement;
+  cacheJsonString?: string;
 };
 
-// ONLY USED IN THIS CLASS 
+// ONLY USED IN THIS CLASS
 const is3DPageMounted = signal(false);
 export const set3DPageMounted = (isMounted: boolean) => {
   is3DPageMounted.value = isMounted;
-}
+};
 export const is3DEditorInitialized = signal(false);
 export const setIs3DEditorInitialized = (isInitialized: boolean) => {
   is3DEditorInitialized.value = isInitialized;
@@ -333,9 +332,7 @@ class Editor {
 
     // set image type at this stage
 
-
     this.renderIndex = 0;
-
   }
 
   // Add helper method to convert focal length to FOV
@@ -400,7 +397,7 @@ class Editor {
   }
   updateSceneContainer(newContainer: HTMLDivElement) {
     this.container = newContainer;
-    this.containerResizeObserver?.disconnect()
+    this.containerResizeObserver?.disconnect();
     this.containerResizeObserver?.observe(this.container);
   }
   engineCanvasMayReset() {
@@ -462,6 +459,7 @@ class Editor {
     editorCanvasEl,
     camViewCanvasEl,
     sceneContainerEl,
+    cacheJsonString: cacheJson = "",
   }: EditorInitializeConfig) {
     if (!this.can_initialize) {
       console.log("Reinitializing 3D editor");
@@ -662,7 +660,9 @@ class Editor {
     // Attach freecam state controller
     this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this);
 
-    if (!this.utils.isEmpty(sceneToken)) {
+    if (!this.utils.isEmpty(cacheJson)) {
+      this.loadCache(cacheJson);
+    } else if (!this.utils.isEmpty(sceneToken)) {
       this.loadScene(sceneToken);
     } else {
       signalScene({
@@ -917,6 +917,10 @@ class Editor {
     }
   }
 
+  public async loadCache(cacheJson: string) {
+    await this.save_manager.loadCache(cacheJson);
+  }
+
   public async loadScene(scene_media_token: string) {
     await this.save_manager.loadScene(scene_media_token);
 
@@ -981,6 +985,20 @@ class Editor {
     });
   }
 
+  public cacheScene({
+    sceneTitle,
+    sceneToken,
+    sceneGenerationMetadata,
+  }: {
+    sceneTitle: string;
+    sceneToken: string;
+    sceneGenerationMetadata: SceneGenereationMetaData;
+  }) {
+    return this.save_manager.getSceneJson({
+      sceneGenerationMetadata: sceneGenerationMetadata,
+    });
+  }
+
   get_camera_person_mode(): boolean {
     return this.camera_person_mode;
   }
@@ -1183,10 +1201,10 @@ class Editor {
             : this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
               ? 576
               : this.render_camera_aspect_ratio ===
-                meraAspectRatio.HORIZONTAL_3_2
+                  meraAspectRatio.HORIZONTAL_3_2
                 ? 900
                 : this.render_camera_aspect_ratio ===
-                  meraAspectRatio.VERTICAL_2_3
+                    meraAspectRatio.VERTICAL_2_3
                   ? 600
                   : 1000;
         const height =
@@ -1195,10 +1213,10 @@ class Editor {
             : this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16
               ? 1024
               : this.render_camera_aspect_ratio ===
-                meraAspectRatio.HORIZONTAL_3_2
+                  meraAspectRatio.HORIZONTAL_3_2
                 ? 600
                 : this.render_camera_aspect_ratio ===
-                  meraAspectRatio.VERTICAL_2_3
+                    meraAspectRatio.VERTICAL_2_3
                   ? 900
                   : 1000;
 
@@ -1261,7 +1279,6 @@ class Editor {
           console.log(`COLLECTING COLOR FRAMES COUNTED: ${this.renderIndex}`);
           await this.engineFrameBuffers.collectColorFrames(this.renderIndex);
 
-
           this.renderIndex = 0;
           console.time("Stop Playback And Upload Video Time");
           await this.stopPlaybackAndUploadVideo();
@@ -1471,6 +1488,13 @@ class Editor {
     this.sceneManager?.detachEventListeners();
     this.disableFreeCamControls();
     this.cameraViewControls?.detachEventListeners();
+
+    // Fix: dispose 3D contexts
+    this.renderer?.dispose();
+    this.composer?.dispose();
+    this.render_composer?.dispose();
+    this.rawRenderer?.dispose();
+
     this.isMounted = false;
     console.log("3D Editor Engine unmounted");
   }

frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/save_manager.ts

@@ -53,8 +53,6 @@ export class SaveManager {
     );
     const scene_json = await proxyScene.saveToScene(this.editor.version);
 
-  
-  
     const check_sum_data = {
       scene: scene_json,
       timeline: "",
@@ -81,8 +79,6 @@ export class SaveManager {
     );
     const scene_json = await proxyScene.saveToScene(this.editor.version);
 
-   
-
     // Save all cameras data
     const camerasData = cameras.value.map((cam: Camera) => ({
       id: cam.id,
@@ -112,28 +108,18 @@ export class SaveManager {
     return jsonString;
   }
 
-  // TODO Move this function into scene manager.
-  public async saveScene({
-    sceneTitle,
-    sceneToken,
+  public getSceneJson({
     sceneGenerationMetadata,
   }: {
-    sceneTitle: string;
-    sceneToken?: string;
     sceneGenerationMetadata: SceneGenereationMetaData;
-  }): Promise<string> {
-    this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out.
-    // remove controls when saving scene.
-    this.editor.utils.removeTransformControls();
-    showEditorLoader();
-
+  }) {
     const proxyScene = new StoryTellerProxyScene(
       this.editor.version,
       this.editor.activeScene,
     );
-    const scene_json = await proxyScene.saveToScene(this.editor.version);
+    const scene_json = proxyScene.saveToScene(this.editor.version);
     console.log(scene_json);
- 
+
     // Save all cameras data
     const camerasData = cameras.value.map((cam: Camera) => ({
       id: cam.id,
@@ -159,6 +145,28 @@ export class SaveManager {
       selectedCameraId: selectedCameraId.value,
     };
 
+    return save_data;
+  }
+
+  // TODO Move this function into scene manager.
+  public async saveScene({
+    sceneTitle,
+    sceneToken,
+    sceneGenerationMetadata,
+  }: {
+    sceneTitle: string;
+    sceneToken?: string;
+    sceneGenerationMetadata: SceneGenereationMetaData;
+  }): Promise<string> {
+    this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out.
+    // remove controls when saving scene.
+    this.editor.utils.removeTransformControls();
+    showEditorLoader();
+
+    const sceneJson = this.getSceneJson({
+      sceneGenerationMetadata,
+    });
+
     // TODO turn scene information into and object ...
     let sceneThumbnail = undefined;
 
@@ -169,7 +177,7 @@ export class SaveManager {
     }
 
     const result = await this.editor.api_manager.saveSceneState({
-      saveJson: JSON.stringify(save_data),
+      saveJson: JSON.stringify(sceneJson),
       sceneTitle,
       sceneToken,
       sceneThumbnail,
@@ -186,19 +194,7 @@ export class SaveManager {
     return result; // if this is an empty string it is an error. need to migrate to api manager.
   }
 
-  // TODO Refactor remove editor.
-  public async loadScene(scene_media_token: string) {
-    showEditorLoader();
-
-    this.editor.current_scene_media_token = scene_media_token;
-
-    const scene_json = await this.editor.api_manager
-      .loadSceneState(this.editor.current_scene_media_token)
-      .catch((err) => {
-        hideEditorLoader();
-        throw err;
-      });
-
+  private async loadFromJson(scene_json: any) {
     const proxyScene = new StoryTellerProxyScene(
       this.editor.version,
       this.editor.activeScene,
@@ -273,6 +269,32 @@ export class SaveManager {
     console.debug("Loading Timeline", scene_json["timeline"]);
 
     this.editor.timeline.checkEditorCanPlay();
+  }
+
+  public async loadCache(cacheJson: string) {
+    showEditorLoader();
+
+    const scene_json = JSON.parse(cacheJson);
+    await this.loadFromJson(scene_json);
+
+    hideEditorLoader();
+    this.editor.timeline.scrub({ currentTime: 0 });
+  }
+
+  // TODO Refactor remove editor.
+  public async loadScene(scene_media_token: string) {
+    showEditorLoader();
+
+    this.editor.current_scene_media_token = scene_media_token;
+
+    const scene_json = await this.editor.api_manager
+      .loadSceneState(this.editor.current_scene_media_token)
+      .catch((err) => {
+        hideEditorLoader();
+        throw err;
+      });
+
+    await this.loadFromJson(scene_json);
 
     hideEditorLoader();
     // TODO figure out if this is a bug.

frontend/apps/editor3d/app/src/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx

@@ -7,11 +7,8 @@ import {
   faQuestion,
   faSquare,
 } from "@fortawesome/pro-solid-svg-icons";
-import {
-  EditorExpandedI,
-  EngineContext,
-} from "~/pages/PageEnigma/contexts/EngineContext";
-import { ToastTypes, getArtStyle } from "~/enums";
+import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
+import { ToastTypes } from "~/enums";
 import { scene, signalScene, authentication, addToast } from "~/signals";
 import { outlinerIsShowing } from "~/pages/PageEnigma/signals/outliner/outliner";
 import { ButtonDropdown } from "@storyteller/ui-button-dropdown";
@@ -20,22 +17,12 @@ import { Button } from "@storyteller/ui-button";
 import { TestFeaturesButtons } from "./TestFeaturesButtons";
 import { LoadUserScenes } from "./LoadUserScenes";
 import { getCurrentLocationWithoutParams, isNumberString } from "~/utilities";
-import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata";
-import {
-  cameraAspectRatio,
-  cinematic,
-  enginePreProcessing,
-  faceDetail,
-  globalIPAMediaToken,
-  lipSync,
-  resetSceneGenerationMetadata,
-  styleStrength,
-  upscale,
-} from "~/pages/PageEnigma/signals";
+import { resetSceneGenerationMetadata } from "~/pages/PageEnigma/signals";
 import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
 import { twMerge } from "tailwind-merge";
 import { Help } from "./Help";
 import { Modal } from "@storyteller/ui-modal";
+import { getSceneGenerationMetaData } from "../../Editor/SceneMetadata";
 
 export const ControlsTopButtons = () => {
   useSignals();
@@ -61,32 +48,12 @@ export const ControlsTopButtons = () => {
     );
   };
 
-  const getSceneGenereationMetaData = useCallback(
-    (editorEngine: EditorExpandedI): SceneGenerationMetaData => {
-      // when this is called, editor engine is guarunteed by it's caller
-      return {
-        positivePrompt: editorEngine.positive_prompt,
-        negativePrompt: editorEngine.negative_prompt,
-        artisticStyle: getArtStyle(editorEngine.art_style.toString()),
-        cameraAspectRatio: cameraAspectRatio.value,
-        globalIPAMediaToken: globalIPAMediaToken.value || undefined,
-        upscale: upscale.value,
-        faceDetail: faceDetail.value,
-        styleStrength: styleStrength.value,
-        lipSync: lipSync.value,
-        cinematic: cinematic.value,
-        enginePreProcessing: enginePreProcessing.value,
-      };
-    },
-    [],
-  );
-
   const handleButtonSave = async () => {
     if (!editorEngine) {
       addToast(ToastTypes.ERROR, "No Engine Error in Saving Scenes");
       return;
     }
-    const sceneGenerationMetadata = getSceneGenereationMetaData(editorEngine);
+    const sceneGenerationMetadata = getSceneGenerationMetaData(editorEngine);
 
     const retSceneMediaToken = await editorEngine.saveScene({
       sceneTitle: scene.value.title || "",
@@ -114,7 +81,7 @@ export const ControlsTopButtons = () => {
       addToast(ToastTypes.ERROR, "No Engine Error in Saving Scenes");
       return;
     }
-    const sceneGenerationMetadata = getSceneGenereationMetaData(editorEngine);
+    const sceneGenerationMetadata = getSceneGenerationMetaData(editorEngine);
     const retSceneMediaToken = await editorEngine.saveScene({
       sceneTitle: sceneTitleInput,
       sceneToken: undefined,
@@ -129,7 +96,7 @@ export const ControlsTopButtons = () => {
         title: sceneTitleInput,
       });
     }
-  }, [sceneTitleInput, editorEngine, getSceneGenereationMetaData]);
+  }, [sceneTitleInput, editorEngine, getSceneGenerationMetaData]);
 
   const handleButtonLoadScene = () => {
     handleResetScene();

frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx

@@ -5,7 +5,8 @@ import Editor from "~/pages/PageEnigma/Editor/editor";
 import { useSignals } from "@preact/signals-react/runtime";
 
 import { signal } from "@preact/signals-react";
-import { useTabStore } from "../../../Stores/TabState"
+import { useTabStore } from "../../../Stores/TabState";
+import { getSceneGenerationMetaData } from "../../Editor/SceneMetadata";
 interface Props {
   sceneToken?: string;
   children: ReactNode;
@@ -23,10 +24,10 @@ export const EngineProvider = ({ sceneToken, children }: Props) => {
 
   const createEditor = () => {
     return new Editor();
-  }
+  };
   const tabStore = useTabStore();
   const tab = tabStore.activeTabId;
-  
+
   const sceneContainer = sceneContainerSignal.value;
   const editorCanvas = editorCanvasSignal.value;
   const camViewCanvas = camViewCanvasSignal.value;
@@ -36,19 +37,40 @@ export const EngineProvider = ({ sceneToken, children }: Props) => {
       if (activeEditorRef.current) {
         return;
       }
-      
+
       const newEditor = createEditor();
       console.warn("Creating new Editor instance", newEditor);
       activeEditorRef.current = newEditor;
+
+      // Check if we have a cached state
+      const cacheString = tabStore.getTabData("3D") as string;
+
       newEditor.initialize({
         sceneToken: sceneToken || "",
         sceneContainerEl: sceneContainer,
         editorCanvasEl: editorCanvas,
         camViewCanvasEl: camViewCanvas,
-      })
+        cacheJsonString: cacheString,
+      });
       setEditor(newEditor);
     } else if (tab !== "3D") {
-      // If the tab is changed, we unmount the editor and clear all input params
+      if (!activeEditorRef.current) {
+        return;
+      }
+
+      // If the tab is changed, cache state, unmount the editor and clear all input params
+      const sceneGenerationMetadata = getSceneGenerationMetaData(
+        activeEditorRef.current,
+      );
+      const cacheJson = activeEditorRef.current.cacheScene({
+        sceneTitle: "",
+        sceneToken: "",
+        sceneGenerationMetadata: sceneGenerationMetadata,
+      });
+      const cacheString = JSON.stringify(cacheJson);
+      tabStore.updateTabData("3D", cacheString);
+
+      // Unmount/Destructor flow
       activeEditorRef.current?.unmountEngine();
       activeEditorRef.current = null;
       sceneContainerSignal.value = null;

frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts

@@ -105,7 +105,7 @@ export class StoryTellerProxy3DObject {
     this.locked = object.userData["locked"];
   }
 
-  public async toJSON(): Promise<ObjectJSON> {
+  public toJSON(): ObjectJSON {
     const json: ObjectJSON = {
       version: this.version,
       position: {

frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_scene.ts

@@ -27,7 +27,7 @@ export class StoryTellerProxyScene {
     this.sceneItemProxy = [];
   }
 
-  async getChildren(child: THREE.Object3D) {
+  getChildren(child: THREE.Object3D) {
     if (this.lookUpDictionary[child.uuid] == null) {
       this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject(
         this.version,
@@ -54,11 +54,11 @@ export class StoryTellerProxyScene {
     const boneHelper = new BoneJSONHelper(child);
     proxyObject3D.rigData = boneHelper.toJSON();
 
-    const json_data = await proxyObject3D.toJSON();
+    const json_data = proxyObject3D.toJSON();
     return json_data;
   }
 
-  public async saveToSceneOlder(): Promise<any> {
+  public saveToSceneOlder(): ObjectJSON[] {
     const results: ObjectJSON[] = [];
     if (this.scene.scene != null) {
       for (const child of this.scene.scene.children) {
@@ -82,7 +82,7 @@ export class StoryTellerProxyScene {
           proxyObject3D.shininess = child.userData["shininess"];
           proxyObject3D.specular = child.userData["specular"];
           proxyObject3D.locked = child.userData["locked"];
-          const json_data = await proxyObject3D.toJSON();
+          const json_data = proxyObject3D.toJSON();
           results.push(json_data);
         }
       }
@@ -93,7 +93,7 @@ export class StoryTellerProxyScene {
     return results;
   }
 
-  public async saveToScene(version: number): Promise<any> {
+  public saveToScene(version: number): ObjectJSON[] {
     this.version = version;
     console.log("Saving with version:", this.version);
     const results: ObjectJSON[] = [];
@@ -101,8 +101,8 @@ export class StoryTellerProxyScene {
       for (const pchild of this.scene.scene.children) {
         if (this.version >= 1.0) {
           if (pchild.userData["media_id"] != undefined) {
-            console.debug("Object JSON:", pchild.toJSON())
-            results.push(await this.getChildren(pchild));
+            console.debug("Object JSON:", pchild.toJSON());
+            results.push(this.getChildren(pchild));
           }
         } else {
           console.log("Saving older.");