From 4cb498111d99c5d79346c2a106a5ae45afe605da Mon Sep 17 00:00:00 2001 From: Tanish Manku Date: Fri, 26 Jun 2026 13:42:32 -0700 Subject: [PATCH] Add experimental scene-descriptor enhancement (manual round-trip) Menu-gated behind a new experimental flag (artcraft_experimental_scene_enhancement). Adds a universal scene descriptor: a compact, LLM-friendly projection of the PageScene that can be exported, edited, and applied back. - scene_descriptor module: versioned descriptor schema, export (scene -> descriptor), apply (descriptor -> scene via existing load path), and glTF/USDZ universal export. - Gray-box: no texture data; entities carry a flat color, default flat-gray shaded. - Mixamo rig poses round-trip losslessly via the per-entity source passthrough. - Enhance panel in the 3D editor (export/copy/download/apply + glTF/USDZ). - experimental-store: new sceneEnhancementEnabled flag, broadcast on change so the pagescene lib reacts via useExperimentalFlag without an inverted lib dependency. Milestone 1 is a manual round-trip; in-app LLM call and editable bone poses deferred. --- .../libs/components/pagescene/src/index.ts | 26 +++ .../pagescene/src/lib/Stage3DBody.tsx | 4 + .../src/lib/comps/SceneDescriptorPanel.tsx | 221 ++++++++++++++++++ .../src/lib/hooks/useExperimentalFlag.ts | 48 ++++ .../src/lib/scene_descriptor/README.md | 61 +++++ .../lib/scene_descriptor/apply_descriptor.ts | 149 ++++++++++++ .../lib/scene_descriptor/export_descriptor.ts | 105 +++++++++ .../src/lib/scene_descriptor/index.ts | 21 ++ .../lib/scene_descriptor/scene_descriptor.ts | 105 +++++++++ .../lib/scene_descriptor/universal_export.ts | 69 ++++++ .../src/lib/experimental-store.ts | 35 ++- .../lib/panes/ExperimentalSettingsPane.tsx | 31 +++ 12 files changed, 874 insertions(+), 1 deletion(-) create mode 100644 frontend/libs/components/pagescene/src/lib/comps/SceneDescriptorPanel.tsx create mode 100644 frontend/libs/components/pagescene/src/lib/hooks/useExperimentalFlag.ts create mode 100644 frontend/libs/components/pagescene/src/lib/scene_descriptor/README.md create mode 100644 frontend/libs/components/pagescene/src/lib/scene_descriptor/apply_descriptor.ts create mode 100644 frontend/libs/components/pagescene/src/lib/scene_descriptor/export_descriptor.ts create mode 100644 frontend/libs/components/pagescene/src/lib/scene_descriptor/index.ts create mode 100644 frontend/libs/components/pagescene/src/lib/scene_descriptor/scene_descriptor.ts create mode 100644 frontend/libs/components/pagescene/src/lib/scene_descriptor/universal_export.ts diff --git a/frontend/libs/components/pagescene/src/index.ts b/frontend/libs/components/pagescene/src/index.ts index 3efaa87a52..d297712fe8 100644 --- a/frontend/libs/components/pagescene/src/index.ts +++ b/frontend/libs/components/pagescene/src/index.ts @@ -181,6 +181,32 @@ export { default as dragAndDrop } from "./lib/DragAndDrop/DndAsset"; // Scene-generation metadata helper (used by save flow + cache snapshot). export { getSceneGenerationMetaData } from "./lib/sceneMetadata"; +// Experimental "scene enhancement" — universal scene descriptor round-trip +// (export → edit/LLM → apply) plus glTF/USDZ universal export. +export { + SCENE_DESCRIPTOR_VERSION, + GRAY_BOX_COLOR, + buildSceneDescriptor, + applySceneDescriptor, + exportSceneToGltf, + exportSceneToUsdz, +} from "./lib/scene_descriptor"; +export type { + SceneDescriptor, + DescriptorEntity, + DescriptorEntityKind, + DescriptorTransform, + DescriptorCamera, + DescriptorEnvironment, + ApplyResult, +} from "./lib/scene_descriptor"; +export { SceneDescriptorPanel } from "./lib/comps/SceneDescriptorPanel"; +export { + useExperimentalFlag, + useSceneEnhancementFlag, + SCENE_ENHANCEMENT_FLAG_KEY, +} from "./lib/hooks/useExperimentalFlag"; + // React UI components. Viewport-dependent comps read viewport size // through useViewportSize, which falls back to window.innerWidth/ // innerHeight when no host adapter is available. diff --git a/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx b/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx index 486c10ab0c..98a44d0c7d 100644 --- a/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx +++ b/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx @@ -52,6 +52,7 @@ import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDispla import { OnboardingHelper } from "./comps/OnboardingHelper"; import { PerfStatsOverlay } from "./comps/PerfStatsOverlay"; import { EntranceDebugPanel } from "./comps/EntranceDebugPanel"; +import { SceneDescriptorPanel } from "./comps/SceneDescriptorPanel"; import { Outliner } from "./comps/Outliner"; import { PoseModeSelector } from "./comps/PoseModeSelector"; import { PreviewBox } from "./comps/PreviewBox"; @@ -63,6 +64,7 @@ import { useFreeCam } from "./hooks/useFreeCam"; import { useViewportPointer } from "./hooks/useViewportPointer"; import { useViewportKeyboard } from "./hooks/useViewportKeyboard"; import { useViewportSize } from "./hooks/useViewportSize"; +import { useSceneEnhancementFlag } from "./hooks/useExperimentalFlag"; import { GridVisibleChangedEvent } from "./engine/events/EngineEvent"; import { pickDropPosition } from "./engine/pickDropPosition"; import { AssetType, CameraAspectRatio } from "./enums"; @@ -212,6 +214,7 @@ export const Stage3DBody = ({ ); const editor = useContext(EngineContext); + const sceneEnhancementEnabled = useSceneEnhancementFlag(); // Reactive viewport sizing. useViewportSize listens to window // resize and re-renders the component. Falls back to @@ -435,6 +438,7 @@ export const Stage3DBody = ({ {import.meta.env.DEV && } + {sceneEnhancementEnabled && } diff --git a/frontend/libs/components/pagescene/src/lib/comps/SceneDescriptorPanel.tsx b/frontend/libs/components/pagescene/src/lib/comps/SceneDescriptorPanel.tsx new file mode 100644 index 0000000000..6aa7e161b5 --- /dev/null +++ b/frontend/libs/components/pagescene/src/lib/comps/SceneDescriptorPanel.tsx @@ -0,0 +1,221 @@ +// Experimental "scene enhancement" panel. Floating control in the 3D +// editor (gated behind the experimental flag) that round-trips the scene +// through a universal descriptor: export the current scene to editable +// JSON, edit it (by hand or via an external LLM), paste it back, and apply. +// Also offers glTF/USDZ export as a universal interchange fallback. +// +// Milestone 1 is a manual round-trip — no in-app LLM call yet. The Apply +// step rebuilds the scene from the pasted descriptor. + +import { useContext, useState } from "react"; +import { EngineContext } from "../contexts/EngineContext/EngineContext"; +import { + applySceneDescriptor, + buildSceneDescriptor, + exportSceneToGltf, + exportSceneToUsdz, + SceneDescriptor, +} from "../scene_descriptor"; + +type Status = { kind: "idle" | "ok" | "err"; message: string }; + +const IDLE: Status = { kind: "idle", message: "" }; + +export function SceneDescriptorPanel() { + const editor = useContext(EngineContext); + const [open, setOpen] = useState(false); + const [text, setText] = useState(""); + const [status, setStatus] = useState(IDLE); + const [busy, setBusy] = useState(false); + + const handleExport = () => { + if (!editor) return; + try { + const descriptor = buildSceneDescriptor(editor); + setText(JSON.stringify(descriptor, null, 2)); + setStatus({ + kind: "ok", + message: `Exported ${descriptor.entities.length} entit${ + descriptor.entities.length === 1 ? "y" : "ies" + }.`, + }); + } catch (err) { + setStatus({ kind: "err", message: errMsg("Export failed", err) }); + } + }; + + const handleCopy = async () => { + if (!text) return; + try { + await navigator.clipboard.writeText(text); + setStatus({ kind: "ok", message: "Copied descriptor to clipboard." }); + } catch (err) { + setStatus({ kind: "err", message: errMsg("Copy failed", err) }); + } + }; + + const handleDownloadJson = () => { + if (!text) return; + downloadBlob( + new Blob([text], { type: "application/json" }), + "scene-descriptor.json", + ); + }; + + const handleDownloadGltf = async () => { + if (!editor) return; + await withBusy(async () => { + const blob = await exportSceneToGltf(editor, { binary: true }); + downloadBlob(blob, "scene.glb"); + setStatus({ kind: "ok", message: "Exported scene.glb." }); + }, "glTF export failed"); + }; + + const handleDownloadUsdz = async () => { + if (!editor) return; + await withBusy(async () => { + const blob = await exportSceneToUsdz(editor); + downloadBlob(blob, "scene.usdz"); + setStatus({ kind: "ok", message: "Exported scene.usdz." }); + }, "USDZ export failed"); + }; + + const handleApply = async () => { + if (!editor || !text.trim()) return; + let descriptor: SceneDescriptor; + try { + descriptor = JSON.parse(text) as SceneDescriptor; + } catch (err) { + setStatus({ kind: "err", message: errMsg("Invalid JSON", err) }); + return; + } + await withBusy(async () => { + const result = await applySceneDescriptor(editor, descriptor); + const skippedNote = result.skipped + ? ` (${result.skipped} skipped — new models/characters need a source)` + : ""; + setStatus({ + kind: "ok", + message: `Applied ${result.applied} entit${ + result.applied === 1 ? "y" : "ies" + }${skippedNote}.`, + }); + }, "Apply failed"); + }; + + const withBusy = async (fn: () => Promise, errLabel: string) => { + setBusy(true); + try { + await fn(); + } catch (err) { + setStatus({ kind: "err", message: errMsg(errLabel, err) }); + } finally { + setBusy(false); + } + }; + + if (!open) { + return ( + + ); + } + + return ( +
+
+ Scene enhancement + +
+ +
+ + Export descriptor + + + Copy + + + Download .json + + + Apply to scene + + + glTF (.glb) + + + USDZ + +
+ +