From 3de468389ec3c00f6102012f33e864aeb76d7ff4 Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Fri, 6 Jun 2025 17:13:31 -0700 Subject: [PATCH] Make 3D state save and load on tab change --- .../pages/PageEnigma/Editor/SceneMetadata.ts | 32 +++++++ .../app/src/pages/PageEnigma/Editor/editor.ts | 50 ++++++++--- .../pages/PageEnigma/Editor/save_manager.ts | 86 ++++++++++++------- .../ControlsTopButtons/ControlsTopButtons.tsx | 47 ++-------- .../contexts/EngineContext/EngineProvider.tsx | 34 ++++++-- .../proxy/storyteller_proxy_3d_object.ts | 2 +- .../proxy/storyteller_proxy_scene.ts | 14 +-- 7 files changed, 166 insertions(+), 99 deletions(-) create mode 100644 frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/SceneMetadata.ts diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/SceneMetadata.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/SceneMetadata.ts new file mode 100644 index 0000000000..b6815a41c3 --- /dev/null +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/SceneMetadata.ts @@ -0,0 +1,32 @@ +import { getArtStyle } from "~/enums/ArtStyle"; +import { + cameraAspectRatio, + cinematic, + enginePreProcessing, + faceDetail, + globalIPAMediaToken, + lipSync, + styleStrength, + upscale, +} from "~/pages/PageEnigma/signals"; +import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; +import Editor from "./editor"; + +export const getSceneGenerationMetaData = ( + editorEngine: Editor, +): SceneGenerationMetaData => { + // when this is called, editor engine is guarunteed by it's caller + return { + positivePrompt: editorEngine.positive_prompt, + negativePrompt: editorEngine.negative_prompt, + artisticStyle: getArtStyle(editorEngine.art_style.toString()), + cameraAspectRatio: cameraAspectRatio.value, + globalIPAMediaToken: globalIPAMediaToken.value || undefined, + upscale: upscale.value, + faceDetail: faceDetail.value, + styleStrength: styleStrength.value, + lipSync: lipSync.value, + cinematic: cinematic.value, + enginePreProcessing: enginePreProcessing.value, + }; +}; diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts index f5dd93b832..a366ffd8cc 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts @@ -50,24 +50,23 @@ import { SceneManager } from "./scene_manager_api"; import { CustomOutlinePass } from "./CustomOutlinePass.js"; import FindSurfaces from "./FindSurfaces.js"; - import Stats from "three/examples/jsm/libs/stats.module.js"; import { CharacterAnimationEngine } from "./Engines/CharacterAnimationEngine"; import { cameras, selectedCameraId } from "~/pages/PageEnigma/signals/camera"; - export type EditorInitializeConfig = { sceneToken: string; editorCanvasEl: HTMLCanvasElement; camViewCanvasEl: HTMLCanvasElement; sceneContainerEl: HTMLDivElement; + cacheJsonString?: string; }; -// ONLY USED IN THIS CLASS +// ONLY USED IN THIS CLASS const is3DPageMounted = signal(false); export const set3DPageMounted = (isMounted: boolean) => { is3DPageMounted.value = isMounted; -} +}; export const is3DEditorInitialized = signal(false); export const setIs3DEditorInitialized = (isInitialized: boolean) => { is3DEditorInitialized.value = isInitialized; @@ -333,9 +332,7 @@ class Editor { // set image type at this stage - this.renderIndex = 0; - } // Add helper method to convert focal length to FOV @@ -400,7 +397,7 @@ class Editor { } updateSceneContainer(newContainer: HTMLDivElement) { this.container = newContainer; - this.containerResizeObserver?.disconnect() + this.containerResizeObserver?.disconnect(); this.containerResizeObserver?.observe(this.container); } engineCanvasMayReset() { @@ -462,6 +459,7 @@ class Editor { editorCanvasEl, camViewCanvasEl, sceneContainerEl, + cacheJsonString: cacheJson = "", }: EditorInitializeConfig) { if (!this.can_initialize) { console.log("Reinitializing 3D editor"); @@ -662,7 +660,9 @@ class Editor { // Attach freecam state controller this.mouseOverEventHandler = this.mouseOverEventHandler.bind(this); - if (!this.utils.isEmpty(sceneToken)) { + if (!this.utils.isEmpty(cacheJson)) { + this.loadCache(cacheJson); + } else if (!this.utils.isEmpty(sceneToken)) { this.loadScene(sceneToken); } else { signalScene({ @@ -917,6 +917,10 @@ class Editor { } } + public async loadCache(cacheJson: string) { + await this.save_manager.loadCache(cacheJson); + } + public async loadScene(scene_media_token: string) { await this.save_manager.loadScene(scene_media_token); @@ -981,6 +985,20 @@ class Editor { }); } + public cacheScene({ + sceneTitle, + sceneToken, + sceneGenerationMetadata, + }: { + sceneTitle: string; + sceneToken: string; + sceneGenerationMetadata: SceneGenereationMetaData; + }) { + return this.save_manager.getSceneJson({ + sceneGenerationMetadata: sceneGenerationMetadata, + }); + } + get_camera_person_mode(): boolean { return this.camera_person_mode; } @@ -1183,10 +1201,10 @@ class Editor { : this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16 ? 576 : this.render_camera_aspect_ratio === - meraAspectRatio.HORIZONTAL_3_2 + meraAspectRatio.HORIZONTAL_3_2 ? 900 : this.render_camera_aspect_ratio === - meraAspectRatio.VERTICAL_2_3 + meraAspectRatio.VERTICAL_2_3 ? 600 : 1000; const height = @@ -1195,10 +1213,10 @@ class Editor { : this.render_camera_aspect_ratio === meraAspectRatio.VERTICAL_9_16 ? 1024 : this.render_camera_aspect_ratio === - meraAspectRatio.HORIZONTAL_3_2 + meraAspectRatio.HORIZONTAL_3_2 ? 600 : this.render_camera_aspect_ratio === - meraAspectRatio.VERTICAL_2_3 + meraAspectRatio.VERTICAL_2_3 ? 900 : 1000; @@ -1261,7 +1279,6 @@ class Editor { console.log(`COLLECTING COLOR FRAMES COUNTED: ${this.renderIndex}`); await this.engineFrameBuffers.collectColorFrames(this.renderIndex); - this.renderIndex = 0; console.time("Stop Playback And Upload Video Time"); await this.stopPlaybackAndUploadVideo(); @@ -1471,6 +1488,13 @@ class Editor { this.sceneManager?.detachEventListeners(); this.disableFreeCamControls(); this.cameraViewControls?.detachEventListeners(); + + // Fix: dispose 3D contexts + this.renderer?.dispose(); + this.composer?.dispose(); + this.render_composer?.dispose(); + this.rawRenderer?.dispose(); + this.isMounted = false; console.log("3D Editor Engine unmounted"); } diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/save_manager.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/save_manager.ts index 124e1e163c..86da0f9de5 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/save_manager.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/save_manager.ts @@ -53,8 +53,6 @@ export class SaveManager { ); const scene_json = await proxyScene.saveToScene(this.editor.version); - - const check_sum_data = { scene: scene_json, timeline: "", @@ -81,8 +79,6 @@ export class SaveManager { ); const scene_json = await proxyScene.saveToScene(this.editor.version); - - // Save all cameras data const camerasData = cameras.value.map((cam: Camera) => ({ id: cam.id, @@ -112,28 +108,18 @@ export class SaveManager { return jsonString; } - // TODO Move this function into scene manager. - public async saveScene({ - sceneTitle, - sceneToken, + public getSceneJson({ sceneGenerationMetadata, }: { - sceneTitle: string; - sceneToken?: string; sceneGenerationMetadata: SceneGenereationMetaData; - }): Promise { - this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out. - // remove controls when saving scene. - this.editor.utils.removeTransformControls(); - showEditorLoader(); - + }) { const proxyScene = new StoryTellerProxyScene( this.editor.version, this.editor.activeScene, ); - const scene_json = await proxyScene.saveToScene(this.editor.version); + const scene_json = proxyScene.saveToScene(this.editor.version); console.log(scene_json); - + // Save all cameras data const camerasData = cameras.value.map((cam: Camera) => ({ id: cam.id, @@ -159,6 +145,28 @@ export class SaveManager { selectedCameraId: selectedCameraId.value, }; + return save_data; + } + + // TODO Move this function into scene manager. + public async saveScene({ + sceneTitle, + sceneToken, + sceneGenerationMetadata, + }: { + sceneTitle: string; + sceneToken?: string; + sceneGenerationMetadata: SceneGenereationMetaData; + }): Promise { + this.editor.generating_preview = true; // Set this to true to stop control panel from flipping out. + // remove controls when saving scene. + this.editor.utils.removeTransformControls(); + showEditorLoader(); + + const sceneJson = this.getSceneJson({ + sceneGenerationMetadata, + }); + // TODO turn scene information into and object ... let sceneThumbnail = undefined; @@ -169,7 +177,7 @@ export class SaveManager { } const result = await this.editor.api_manager.saveSceneState({ - saveJson: JSON.stringify(save_data), + saveJson: JSON.stringify(sceneJson), sceneTitle, sceneToken, sceneThumbnail, @@ -186,19 +194,7 @@ export class SaveManager { return result; // if this is an empty string it is an error. need to migrate to api manager. } - // TODO Refactor remove editor. - public async loadScene(scene_media_token: string) { - showEditorLoader(); - - this.editor.current_scene_media_token = scene_media_token; - - const scene_json = await this.editor.api_manager - .loadSceneState(this.editor.current_scene_media_token) - .catch((err) => { - hideEditorLoader(); - throw err; - }); - + private async loadFromJson(scene_json: any) { const proxyScene = new StoryTellerProxyScene( this.editor.version, this.editor.activeScene, @@ -273,6 +269,32 @@ export class SaveManager { console.debug("Loading Timeline", scene_json["timeline"]); this.editor.timeline.checkEditorCanPlay(); + } + + public async loadCache(cacheJson: string) { + showEditorLoader(); + + const scene_json = JSON.parse(cacheJson); + await this.loadFromJson(scene_json); + + hideEditorLoader(); + this.editor.timeline.scrub({ currentTime: 0 }); + } + + // TODO Refactor remove editor. + public async loadScene(scene_media_token: string) { + showEditorLoader(); + + this.editor.current_scene_media_token = scene_media_token; + + const scene_json = await this.editor.api_manager + .loadSceneState(this.editor.current_scene_media_token) + .catch((err) => { + hideEditorLoader(); + throw err; + }); + + await this.loadFromJson(scene_json); hideEditorLoader(); // TODO figure out if this is a bug. diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx index c88a505912..d85d0ac8e4 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/ControlsTopButtons/ControlsTopButtons.tsx @@ -7,11 +7,8 @@ import { faQuestion, faSquare, } from "@fortawesome/pro-solid-svg-icons"; -import { - EditorExpandedI, - EngineContext, -} from "~/pages/PageEnigma/contexts/EngineContext"; -import { ToastTypes, getArtStyle } from "~/enums"; +import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; +import { ToastTypes } from "~/enums"; import { scene, signalScene, authentication, addToast } from "~/signals"; import { outlinerIsShowing } from "~/pages/PageEnigma/signals/outliner/outliner"; import { ButtonDropdown } from "@storyteller/ui-button-dropdown"; @@ -20,22 +17,12 @@ import { Button } from "@storyteller/ui-button"; import { TestFeaturesButtons } from "./TestFeaturesButtons"; import { LoadUserScenes } from "./LoadUserScenes"; import { getCurrentLocationWithoutParams, isNumberString } from "~/utilities"; -import { SceneGenereationMetaData as SceneGenerationMetaData } from "~/pages/PageEnigma/models/sceneGenerationMetadata"; -import { - cameraAspectRatio, - cinematic, - enginePreProcessing, - faceDetail, - globalIPAMediaToken, - lipSync, - resetSceneGenerationMetadata, - styleStrength, - upscale, -} from "~/pages/PageEnigma/signals"; +import { resetSceneGenerationMetadata } from "~/pages/PageEnigma/signals"; import { CameraAspectRatio } from "~/pages/PageEnigma/enums"; import { twMerge } from "tailwind-merge"; import { Help } from "./Help"; import { Modal } from "@storyteller/ui-modal"; +import { getSceneGenerationMetaData } from "../../Editor/SceneMetadata"; export const ControlsTopButtons = () => { useSignals(); @@ -61,32 +48,12 @@ export const ControlsTopButtons = () => { ); }; - const getSceneGenereationMetaData = useCallback( - (editorEngine: EditorExpandedI): SceneGenerationMetaData => { - // when this is called, editor engine is guarunteed by it's caller - return { - positivePrompt: editorEngine.positive_prompt, - negativePrompt: editorEngine.negative_prompt, - artisticStyle: getArtStyle(editorEngine.art_style.toString()), - cameraAspectRatio: cameraAspectRatio.value, - globalIPAMediaToken: globalIPAMediaToken.value || undefined, - upscale: upscale.value, - faceDetail: faceDetail.value, - styleStrength: styleStrength.value, - lipSync: lipSync.value, - cinematic: cinematic.value, - enginePreProcessing: enginePreProcessing.value, - }; - }, - [], - ); - const handleButtonSave = async () => { if (!editorEngine) { addToast(ToastTypes.ERROR, "No Engine Error in Saving Scenes"); return; } - const sceneGenerationMetadata = getSceneGenereationMetaData(editorEngine); + const sceneGenerationMetadata = getSceneGenerationMetaData(editorEngine); const retSceneMediaToken = await editorEngine.saveScene({ sceneTitle: scene.value.title || "", @@ -114,7 +81,7 @@ export const ControlsTopButtons = () => { addToast(ToastTypes.ERROR, "No Engine Error in Saving Scenes"); return; } - const sceneGenerationMetadata = getSceneGenereationMetaData(editorEngine); + const sceneGenerationMetadata = getSceneGenerationMetaData(editorEngine); const retSceneMediaToken = await editorEngine.saveScene({ sceneTitle: sceneTitleInput, sceneToken: undefined, @@ -129,7 +96,7 @@ export const ControlsTopButtons = () => { title: sceneTitleInput, }); } - }, [sceneTitleInput, editorEngine, getSceneGenereationMetaData]); + }, [sceneTitleInput, editorEngine, getSceneGenerationMetaData]); const handleButtonLoadScene = () => { handleResetScene(); diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx index 8d53205a16..4b9a3b08d9 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/contexts/EngineContext/EngineProvider.tsx @@ -5,7 +5,8 @@ import Editor from "~/pages/PageEnigma/Editor/editor"; import { useSignals } from "@preact/signals-react/runtime"; import { signal } from "@preact/signals-react"; -import { useTabStore } from "../../../Stores/TabState" +import { useTabStore } from "../../../Stores/TabState"; +import { getSceneGenerationMetaData } from "../../Editor/SceneMetadata"; interface Props { sceneToken?: string; children: ReactNode; @@ -23,10 +24,10 @@ export const EngineProvider = ({ sceneToken, children }: Props) => { const createEditor = () => { return new Editor(); - } + }; const tabStore = useTabStore(); const tab = tabStore.activeTabId; - + const sceneContainer = sceneContainerSignal.value; const editorCanvas = editorCanvasSignal.value; const camViewCanvas = camViewCanvasSignal.value; @@ -36,19 +37,40 @@ export const EngineProvider = ({ sceneToken, children }: Props) => { if (activeEditorRef.current) { return; } - + const newEditor = createEditor(); console.warn("Creating new Editor instance", newEditor); activeEditorRef.current = newEditor; + + // Check if we have a cached state + const cacheString = tabStore.getTabData("3D") as string; + newEditor.initialize({ sceneToken: sceneToken || "", sceneContainerEl: sceneContainer, editorCanvasEl: editorCanvas, camViewCanvasEl: camViewCanvas, - }) + cacheJsonString: cacheString, + }); setEditor(newEditor); } else if (tab !== "3D") { - // If the tab is changed, we unmount the editor and clear all input params + if (!activeEditorRef.current) { + return; + } + + // If the tab is changed, cache state, unmount the editor and clear all input params + const sceneGenerationMetadata = getSceneGenerationMetaData( + activeEditorRef.current, + ); + const cacheJson = activeEditorRef.current.cacheScene({ + sceneTitle: "", + sceneToken: "", + sceneGenerationMetadata: sceneGenerationMetadata, + }); + const cacheString = JSON.stringify(cacheJson); + tabStore.updateTabData("3D", cacheString); + + // Unmount/Destructor flow activeEditorRef.current?.unmountEngine(); activeEditorRef.current = null; sceneContainerSignal.value = null; diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts index a067a8817d..ed076f8acd 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_3d_object.ts @@ -105,7 +105,7 @@ export class StoryTellerProxy3DObject { this.locked = object.userData["locked"]; } - public async toJSON(): Promise { + public toJSON(): ObjectJSON { const json: ObjectJSON = { version: this.version, position: { diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_scene.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_scene.ts index 4561cee3ec..d51471aae3 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_scene.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/proxy/storyteller_proxy_scene.ts @@ -27,7 +27,7 @@ export class StoryTellerProxyScene { this.sceneItemProxy = []; } - async getChildren(child: THREE.Object3D) { + getChildren(child: THREE.Object3D) { if (this.lookUpDictionary[child.uuid] == null) { this.lookUpDictionary[child.uuid] = new StoryTellerProxy3DObject( this.version, @@ -54,11 +54,11 @@ export class StoryTellerProxyScene { const boneHelper = new BoneJSONHelper(child); proxyObject3D.rigData = boneHelper.toJSON(); - const json_data = await proxyObject3D.toJSON(); + const json_data = proxyObject3D.toJSON(); return json_data; } - public async saveToSceneOlder(): Promise { + public saveToSceneOlder(): ObjectJSON[] { const results: ObjectJSON[] = []; if (this.scene.scene != null) { for (const child of this.scene.scene.children) { @@ -82,7 +82,7 @@ export class StoryTellerProxyScene { proxyObject3D.shininess = child.userData["shininess"]; proxyObject3D.specular = child.userData["specular"]; proxyObject3D.locked = child.userData["locked"]; - const json_data = await proxyObject3D.toJSON(); + const json_data = proxyObject3D.toJSON(); results.push(json_data); } } @@ -93,7 +93,7 @@ export class StoryTellerProxyScene { return results; } - public async saveToScene(version: number): Promise { + public saveToScene(version: number): ObjectJSON[] { this.version = version; console.log("Saving with version:", this.version); const results: ObjectJSON[] = []; @@ -101,8 +101,8 @@ export class StoryTellerProxyScene { for (const pchild of this.scene.scene.children) { if (this.version >= 1.0) { if (pchild.userData["media_id"] != undefined) { - console.debug("Object JSON:", pchild.toJSON()) - results.push(await this.getChildren(pchild)); + console.debug("Object JSON:", pchild.toJSON()); + results.push(this.getChildren(pchild)); } } else { console.log("Saving older.");