Add experimental scene-descriptor enhancement (manual round-trip)

Menu-gated behind a new experimental flag (artcraft_experimental_scene_enhancement).
Adds a universal scene descriptor: a compact, LLM-friendly projection of the
PageScene that can be exported, edited, and applied back.

- scene_descriptor module: versioned descriptor schema, export (scene -> descriptor),
  apply (descriptor -> scene via existing load path), and glTF/USDZ universal export.
- Gray-box: no texture data; entities carry a flat color, default flat-gray shaded.
- Mixamo rig poses round-trip losslessly via the per-entity source passthrough.
- Enhance panel in the 3D editor (export/copy/download/apply + glTF/USDZ).
- experimental-store: new sceneEnhancementEnabled flag, broadcast on change so the
  pagescene lib reacts via useExperimentalFlag without an inverted lib dependency.

Milestone 1 is a manual round-trip; in-app LLM call and editable bone poses deferred.
This commit is contained in:
Tanish Manku committed 2026-06-26 13:42:32 -07:00
1 parent 4d5f8e9c21
commit 4cb498111d
12 files changed
+874 -1

No files matched your search

@@ -181,6 +181,32 @@ export { default as dragAndDrop } from "./lib/DragAndDrop/DndAsset";
// Scene-generation metadata helper (used by save flow + cache snapshot).
export { getSceneGenerationMetaData } from "./lib/sceneMetadata";
// Experimental "scene enhancement" — universal scene descriptor round-trip
// (export → edit/LLM → apply) plus glTF/USDZ universal export.
export {
SCENE_DESCRIPTOR_VERSION,
GRAY_BOX_COLOR,
buildSceneDescriptor,
applySceneDescriptor,
exportSceneToGltf,
exportSceneToUsdz,
} from "./lib/scene_descriptor";
export type {
SceneDescriptor,
DescriptorEntity,
DescriptorEntityKind,
DescriptorTransform,
DescriptorCamera,
DescriptorEnvironment,
ApplyResult,
} from "./lib/scene_descriptor";
export { SceneDescriptorPanel } from "./lib/comps/SceneDescriptorPanel";
export {
useExperimentalFlag,
useSceneEnhancementFlag,
SCENE_ENHANCEMENT_FLAG_KEY,
} from "./lib/hooks/useExperimentalFlag";
// React UI components. Viewport-dependent comps read viewport size
// through useViewportSize, which falls back to window.innerWidth/
// innerHeight when no host adapter is available.
@@ -52,6 +52,7 @@ import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDispla
import { OnboardingHelper } from "./comps/OnboardingHelper";
import { PerfStatsOverlay } from "./comps/PerfStatsOverlay";
import { EntranceDebugPanel } from "./comps/EntranceDebugPanel";
import { SceneDescriptorPanel } from "./comps/SceneDescriptorPanel";
import { Outliner } from "./comps/Outliner";
import { PoseModeSelector } from "./comps/PoseModeSelector";
import { PreviewBox } from "./comps/PreviewBox";
@@ -63,6 +64,7 @@ import { useFreeCam } from "./hooks/useFreeCam";
import { useViewportPointer } from "./hooks/useViewportPointer";
import { useViewportKeyboard } from "./hooks/useViewportKeyboard";
import { useViewportSize } from "./hooks/useViewportSize";
import { useSceneEnhancementFlag } from "./hooks/useExperimentalFlag";
import { GridVisibleChangedEvent } from "./engine/events/EngineEvent";
import { pickDropPosition } from "./engine/pickDropPosition";
import { AssetType, CameraAspectRatio } from "./enums";
@@ -212,6 +214,7 @@ export const Stage3DBody = ({
);
const editor = useContext(EngineContext);
const sceneEnhancementEnabled = useSceneEnhancementFlag();
// Reactive viewport sizing. useViewportSize listens to window
// resize and re-renders the component. Falls back to
@@ -435,6 +438,7 @@ export const Stage3DBody = ({
<PerfStatsOverlay />
{import.meta.env.DEV && <EntranceDebugPanel />}
{sceneEnhancementEnabled && <SceneDescriptorPanel />}
<FocalLengthDisplay />
<PoseModeSelector />
@@ -0,0 +1,221 @@
// Experimental "scene enhancement" panel. Floating control in the 3D
// editor (gated behind the experimental flag) that round-trips the scene
// through a universal descriptor: export the current scene to editable
// JSON, edit it (by hand or via an external LLM), paste it back, and apply.
// Also offers glTF/USDZ export as a universal interchange fallback.
//
// Milestone 1 is a manual round-trip — no in-app LLM call yet. The Apply
// step rebuilds the scene from the pasted descriptor.
import { useContext, useState } from "react";
import { EngineContext } from "../contexts/EngineContext/EngineContext";
import {
applySceneDescriptor,
buildSceneDescriptor,
exportSceneToGltf,
exportSceneToUsdz,
SceneDescriptor,
} from "../scene_descriptor";
type Status = { kind: "idle" | "ok" | "err"; message: string };
const IDLE: Status = { kind: "idle", message: "" };
export function SceneDescriptorPanel() {
const editor = useContext(EngineContext);
const [open, setOpen] = useState(false);
const [text, setText] = useState("");
const [status, setStatus] = useState<Status>(IDLE);
const [busy, setBusy] = useState(false);
const handleExport = () => {
if (!editor) return;
try {
const descriptor = buildSceneDescriptor(editor);
setText(JSON.stringify(descriptor, null, 2));
setStatus({
kind: "ok",
message: `Exported ${descriptor.entities.length} entit${
descriptor.entities.length === 1 ? "y" : "ies"
}.`,
});
} catch (err) {
setStatus({ kind: "err", message: errMsg("Export failed", err) });
}
};
const handleCopy = async () => {
if (!text) return;
try {
await navigator.clipboard.writeText(text);
setStatus({ kind: "ok", message: "Copied descriptor to clipboard." });
} catch (err) {
setStatus({ kind: "err", message: errMsg("Copy failed", err) });
}
};
const handleDownloadJson = () => {
if (!text) return;
downloadBlob(
new Blob([text], { type: "application/json" }),
"scene-descriptor.json",
);
};
const handleDownloadGltf = async () => {
if (!editor) return;
await withBusy(async () => {
const blob = await exportSceneToGltf(editor, { binary: true });
downloadBlob(blob, "scene.glb");
setStatus({ kind: "ok", message: "Exported scene.glb." });
}, "glTF export failed");
};
const handleDownloadUsdz = async () => {
if (!editor) return;
await withBusy(async () => {
const blob = await exportSceneToUsdz(editor);
downloadBlob(blob, "scene.usdz");
setStatus({ kind: "ok", message: "Exported scene.usdz." });
}, "USDZ export failed");
};
const handleApply = async () => {
if (!editor || !text.trim()) return;
let descriptor: SceneDescriptor;
try {
descriptor = JSON.parse(text) as SceneDescriptor;
} catch (err) {
setStatus({ kind: "err", message: errMsg("Invalid JSON", err) });
return;
}
await withBusy(async () => {
const result = await applySceneDescriptor(editor, descriptor);
const skippedNote = result.skipped
? ` (${result.skipped} skipped — new models/characters need a source)`
: "";
setStatus({
kind: "ok",
message: `Applied ${result.applied} entit${
result.applied === 1 ? "y" : "ies"
}${skippedNote}.`,
});
}, "Apply failed");
};
const withBusy = async (fn: () => Promise<void>, errLabel: string) => {
setBusy(true);
try {
await fn();
} catch (err) {
setStatus({ kind: "err", message: errMsg(errLabel, err) });
} finally {
setBusy(false);
}
};
if (!open) {
return (
<button
type="button"
onClick={() => setOpen(true)}
className="pointer-events-auto absolute right-2 top-16 z-50 rounded-lg border border-white/10 bg-black/70 px-3 py-1.5 text-xs font-medium text-white/80 backdrop-blur-md transition-colors hover:bg-white/10 hover:text-white/90"
>
✦ Enhance
</button>
);
}
return (
<div className="pointer-events-auto absolute right-2 top-16 z-50 flex w-72 flex-col gap-2 rounded-lg border border-white/10 bg-black/70 p-3 text-xs text-white/80 backdrop-blur-md">
<div className="flex items-center justify-between font-medium text-white/90">
<span>Scene enhancement</span>
<button
type="button"
onClick={() => setOpen(false)}
className="rounded px-1 text-white/50 hover:text-white/90"
aria-label="Close"
>
✕
</button>
</div>
<div className="grid grid-cols-2 gap-1.5">
<PanelButton onClick={handleExport} disabled={!editor || busy}>
Export descriptor
</PanelButton>
<PanelButton onClick={handleCopy} disabled={!text || busy}>
Copy
</PanelButton>
<PanelButton onClick={handleDownloadJson} disabled={!text || busy}>
Download .json
</PanelButton>
<PanelButton onClick={handleApply} disabled={!editor || !text || busy}>
Apply to scene
</PanelButton>
<PanelButton onClick={handleDownloadGltf} disabled={!editor || busy}>
glTF (.glb)
</PanelButton>
<PanelButton onClick={handleDownloadUsdz} disabled={!editor || busy}>
USDZ
</PanelButton>
</div>
<textarea
value={text}
onChange={(e) => setText(e.target.value)}
spellCheck={false}
placeholder="Export a descriptor, edit it (or paste an LLM-edited one), then Apply to scene."
className="h-40 w-full resize-y rounded border border-white/10 bg-black/50 p-2 font-mono text-[10px] leading-snug text-white/80 outline-none focus:border-white/30"
/>
{status.kind !== "idle" && (
<p
className={
status.kind === "err" ? "text-red-300" : "text-emerald-300"
}
>
{status.message}
</p>
)}
</div>
);
}
function PanelButton({
onClick,
disabled,
children,
}: {
onClick: () => void;
disabled?: boolean;
children: React.ReactNode;
}) {
return (
<button
type="button"
onClick={onClick}
disabled={disabled}
className="rounded border border-white/10 py-1 text-white/70 transition-colors hover:bg-white/10 hover:text-white/90 disabled:cursor-not-allowed disabled:opacity-40"
>
{children}
</button>
);
}
function downloadBlob(blob: Blob, filename: string) {
const url = URL.createObjectURL(blob);
const link = document.createElement("a");
link.href = url;
link.download = filename;
document.body.appendChild(link);
link.click();
link.remove();
// Revoke on the next tick so the click-driven navigation has started.
setTimeout(() => URL.revokeObjectURL(url), 0);
}
function errMsg(label: string, err: unknown): string {
const detail = err instanceof Error ? err.message : String(err);
return `${label}: ${detail}`;
}
@@ -0,0 +1,48 @@
// Read an experimental feature flag without importing the settings-modal
// store (that lib sits above pagescene in the layering, so we can't depend
// on it). The flag's source of truth is localStorage; the settings store
// broadcasts an `artcraft_experimental_changed` CustomEvent on every
// toggle so we can refresh same-tab. The cross-tab `storage` event covers
// other tabs. The string keys + event name are the shared contract with
// experimental-store.ts.
import { useEffect, useState } from "react";
const MASTER_KEY = "artcraft_experimental_enabled";
const FLAGS_CHANGED_EVENT = "artcraft_experimental_changed";
const read = (featureKey: string): boolean => {
if (typeof window === "undefined") return false;
try {
return (
window.localStorage.getItem(MASTER_KEY) === "true" &&
window.localStorage.getItem(featureKey) === "true"
);
} catch {
return false;
}
};
export function useExperimentalFlag(featureKey: string): boolean {
const [enabled, setEnabled] = useState(() => read(featureKey));
useEffect(() => {
const refresh = () => setEnabled(read(featureKey));
refresh();
window.addEventListener(FLAGS_CHANGED_EVENT, refresh);
window.addEventListener("storage", refresh);
return () => {
window.removeEventListener(FLAGS_CHANGED_EVENT, refresh);
window.removeEventListener("storage", refresh);
};
}, [featureKey]);
return enabled;
}
// The scene-enhancement feature gate.
export const SCENE_ENHANCEMENT_FLAG_KEY =
"artcraft_experimental_scene_enhancement";
export const useSceneEnhancementFlag = () =>
useExperimentalFlag(SCENE_ENHANCEMENT_FLAG_KEY);
@@ -0,0 +1,61 @@
# Scene Descriptor (experimental "scene enhancement")
A compact, LLM-friendly projection of the live PageScene that can be
exported, edited, and applied back — the foundation for using an LLM to
iterate the contents of a 3D scene.
This is **menu-gated** behind the experimental flag
`artcraft_experimental_scene_enhancement` (Settings → Experimental →
"Scene enhancement"). When on, an **✦ Enhance** panel appears in the 3D
editor.
## Round-trip
```
buildSceneDescriptor(editor) → SceneDescriptor (JSON)
↓ edit by hand, or paste through any external LLM
applySceneDescriptor(editor, descriptor) → scene rebuilt
```
Milestone 1 is a **manual** round-trip — no in-app LLM call yet. Export the
descriptor, edit/enhance it however you like, paste it back, Apply.
## Format
`SceneDescriptor` (see `scene_descriptor.ts`) is a lossy, editable *view*
of the scene:
- `entities[]` — one per scene object with `id`, `name`, `kind`
(`primitive | model | character | light | image | point`), optional
`shape`/`color`/`visible`, and a `transform` (`position`, `rotationDeg`
in **degrees**, `scale`).
- `environment.skybox`, plus a read-only `camera` for spatial context.
Each entity also carries `source`: the full internal `ObjectJSON`. This is
**not** for hand-editing — it's the lossless passthrough (asset token,
material, mixamo rig pose) that lets a round-trip reconstruct an object
exactly. An entity with no `source` is treated as new; only **primitives**
can be synthesized from scratch (models/characters need their asset token).
### Gray-box
Textures are intentionally not represented. Entities carry a flat `color`
only; anything without one renders flat-gray (`GRAY_BOX_COLOR`). An LLM can
add and recolor primitives without ever touching texture data.
## Universal export (fallback)
`exportSceneToGltf` / `exportSceneToUsdz` export the scene to glTF/GLB and
USDZ — the portable interchange formats, alongside the JSON descriptor.
## Roadmap / not done yet
- **Editable mixamo pose** — bone poses currently round-trip losslessly via
`source.rigData` but are not yet decoded into a per-bone editable surface
(`hasPose` advertises their presence). Decode/encode is the next step so
an LLM can re-pose characters.
- **In-app LLM call** — placement (backend `storyteller_web` endpoint vs.
frontend/Tauri) is deliberately deferred; milestone 1 is manual.
- **Non-destructive apply** — Apply currently rebuilds the scene and clears
the undo stack (same path as "Reset to original"). A diff-based apply
(move/add/remove in place, undoable) is a follow-up.
@@ -0,0 +1,149 @@
// Apply: SceneDescriptor → live PageScene.
//
// We translate the descriptor back into the ObjectJSON[] the editor's
// existing load path already understands, then hand it to
// editor.applyJson (which clears the undo stack and rebuilds the scene).
// Existing entities are reconstructed from their lossless `source`;
// entities the editor never saw (e.g. an LLM-authored primitive) are
// synthesized from the editable fields.
import * as THREE from "three";
import { v4 as uuidv4 } from "uuid";
import type Editor from "../engine/editor";
import { ObjectJSON } from "../proxy/storyteller_proxy_3d_object";
import { MediaFileType } from "../enums";
import {
DescriptorEntity,
GRAY_BOX_COLOR,
SceneDescriptor,
Vec3,
} from "./scene_descriptor";
export interface ApplyResult {
applied: number;
skipped: number;
}
export async function applySceneDescriptor(
editor: Editor,
descriptor: SceneDescriptor,
): Promise<ApplyResult> {
const entities = descriptor?.entities ?? [];
let skipped = 0;
const sceneJson: ObjectJSON[] = [];
for (const entity of entities) {
const obj = entityToObjectJson(entity, editor.version);
if (obj) {
sceneJson.push(obj);
} else {
skipped++;
}
}
const wrapper = {
version: editor.version,
scene: sceneJson,
skybox: descriptor?.environment?.skybox ?? editor.activeScene.skybox ?? "",
};
await editor.applyJson(JSON.stringify(wrapper));
return { applied: sceneJson.length, skipped };
}
// Build an ObjectJSON for a single entity. Returns undefined when the
// entity can't be reconstructed (a new model/character with no `source`,
// since the asset token is unknown).
function entityToObjectJson(
entity: DescriptorEntity,
version: number,
): ObjectJSON | undefined {
const base = entity.source
? cloneObjectJson(entity.source)
: synthesize(entity, version);
if (!base) return undefined;
// Overlay the editable surface onto the (possibly preserved) base.
base.object_uuid = entity.id || base.object_uuid || uuidv4();
if (entity.name) {
base.object_user_data_name = entity.name;
base.user_data = { ...base.user_data, name: entity.name };
}
if (entity.color) base.color = entity.color;
if (entity.visible !== undefined) base.visible = entity.visible;
const t = entity.transform;
if (t) {
base.position = vec(t.position);
base.rotation = eulerDegToRad(t.rotationDeg);
base.scale = vec(t.scale, 1);
}
// A primitive whose shape changed needs the geometry key updated so the
// loader's instantiate() picks the new geometry on rebuild.
if (base.media_file_token === "Parim" && entity.shape) {
base.object_name = entity.shape;
base.user_data = {
...base.user_data,
shapeKey: entity.shape,
shapeType: entity.shape,
isShape: true,
};
}
return base;
}
// Synthesize an ObjectJSON for an entity that has no `source`. Only
// primitives are supported — models/characters require an asset token we
// can't invent.
function synthesize(
entity: DescriptorEntity,
version: number,
): ObjectJSON | undefined {
if (entity.kind !== "primitive" || !entity.shape) return undefined;
const shape = entity.shape;
return {
version,
position: { x: 0, y: 0, z: 0 },
rotation: { x: 0, y: 0, z: 0 },
scale: { x: 1, y: 1, z: 1 },
object_name: shape,
object_uuid: entity.id || uuidv4(),
object_user_data_name: entity.name || shape,
media_file_token: "Parim",
color: entity.color || GRAY_BOX_COLOR,
metalness: 0,
shininess: 0.5,
specular: 0.5,
locked: false,
visible: entity.visible ?? true,
rigData: undefined,
user_data: {
name: entity.name || shape,
isShape: true,
shapeKey: shape,
shapeType: shape,
media_file_type: MediaFileType.None,
},
};
}
// Deep-clone so we never mutate the descriptor the caller still holds.
function cloneObjectJson(obj: ObjectJSON): ObjectJSON {
return JSON.parse(JSON.stringify(obj));
}
function vec(v: Vec3 | undefined, fallback = 0): { x: number; y: number; z: number } {
if (!v) return { x: fallback, y: fallback, z: fallback };
return { x: v.x ?? fallback, y: v.y ?? fallback, z: v.z ?? fallback };
}
function eulerDegToRad(v: Vec3 | undefined): { x: number; y: number; z: number } {
if (!v) return { x: 0, y: 0, z: 0 };
return {
x: THREE.MathUtils.degToRad(v.x ?? 0),
y: THREE.MathUtils.degToRad(v.y ?? 0),
z: THREE.MathUtils.degToRad(v.z ?? 0),
};
}
@@ -0,0 +1,105 @@
// Export: live PageScene → SceneDescriptor.
//
// We lean on the existing serialization machinery (StoryTellerProxyScene)
// to snapshot the scene into ObjectJSON[], then project each object into
// the compact, editable descriptor entity while stashing the full
// ObjectJSON in `source` for a lossless round-trip.
import * as THREE from "three";
import type Editor from "../engine/editor";
import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene";
import { ObjectJSON } from "../proxy/storyteller_proxy_3d_object";
import {
DescriptorCamera,
DescriptorEntity,
DescriptorEntityKind,
SceneDescriptor,
SCENE_DESCRIPTOR_VERSION,
Vec3,
} from "./scene_descriptor";
export function buildSceneDescriptor(editor: Editor): SceneDescriptor {
const scene = editor.activeScene;
const proxy = new StoryTellerProxyScene(editor.version, scene);
const objects = proxy.saveToScene(editor.version);
const entities = objects.map(objectJsonToEntity);
return {
descriptorVersion: SCENE_DESCRIPTOR_VERSION,
units: "meters",
coordinateSystem: "y-up-right-handed",
environment: { skybox: scene.skybox || undefined },
camera: readCamera(editor),
entities,
};
}
function objectJsonToEntity(obj: ObjectJSON): DescriptorEntity {
const kind = classifyKind(obj);
const entity: DescriptorEntity = {
id: obj.object_uuid,
name: obj.object_user_data_name || obj.object_name || "",
kind,
color: obj.color || undefined,
visible: obj.visible !== false,
transform: {
position: radSafeVec(obj.position),
rotationDeg: eulerRadToDeg(obj.rotation),
scale: radSafeVec(obj.scale),
},
source: obj,
};
if (kind === "primitive") {
entity.shape =
(obj.user_data?.shapeType as string | undefined) ??
(obj.user_data?.shapeKey as string | undefined) ??
obj.object_name;
}
if (obj.rigData) {
entity.hasPose = true;
}
return entity;
}
// Mirror of the token dispatch in StoryTellerProxyScene.loadFromSceneJson:
// the media_file_token is the discriminator the loader switches on.
function classifyKind(obj: ObjectJSON): DescriptorEntityKind {
const token = obj.media_file_token ?? "";
if (token === "Parim" || obj.user_data?.isShape === true) return "primitive";
if (token === "DirectionalLight") return "light";
if (token.includes("Point::")) return "point";
if (token.includes("Image::")) return "image";
if (token.includes("m_")) {
return obj.user_data?.isCharacter === true ? "character" : "model";
}
return "unknown";
}
function readCamera(editor: Editor): DescriptorCamera | undefined {
const cam = editor.cameraController?.camera;
if (!cam) return undefined;
return {
position: { x: cam.position.x, y: cam.position.y, z: cam.position.z },
rotationDeg: eulerRadToDeg(cam.rotation),
};
}
function eulerRadToDeg(r: { x: number; y: number; z: number }): Vec3 {
return {
x: round(THREE.MathUtils.radToDeg(r.x)),
y: round(THREE.MathUtils.radToDeg(r.y)),
z: round(THREE.MathUtils.radToDeg(r.z)),
};
}
function radSafeVec(v: { x: number; y: number; z: number }): Vec3 {
return { x: round(v.x), y: round(v.y), z: round(v.z) };
}
// Trim float noise so the exported JSON reads cleanly. 5 decimals keeps
// sub-millimeter precision while dropping 1e-16 dust from THREE math.
function round(n: number): number {
return Math.round((n + Number.EPSILON) * 1e5) / 1e5;
}
@@ -0,0 +1,21 @@
// Experimental "scene enhancement" — universal scene descriptor.
// Round-trip: buildSceneDescriptor → (edit / LLM) → applySceneDescriptor.
export {
SCENE_DESCRIPTOR_VERSION,
GRAY_BOX_COLOR,
} from "./scene_descriptor";
export type {
SceneDescriptor,
DescriptorEntity,
DescriptorEntityKind,
DescriptorTransform,
DescriptorCamera,
DescriptorEnvironment,
Vec3,
} from "./scene_descriptor";
export { buildSceneDescriptor } from "./export_descriptor";
export { applySceneDescriptor } from "./apply_descriptor";
export type { ApplyResult } from "./apply_descriptor";
export { exportSceneToGltf, exportSceneToUsdz } from "./universal_export";
@@ -0,0 +1,105 @@
// Universal Scene Descriptor — a compact, LLM-friendly projection of the
// live 3D scene.
//
// This is the editable interchange format for the experimental
// "scene enhancement" feature: we export the PageScene into a descriptor,
// let a human (or, later, an LLM) edit it, then apply it back to update
// the scene. It is intentionally a *lossy view* of the heavy internal
// `ObjectJSON` — only the spatial/semantic fields a model can reason
// about are surfaced at the top level. Everything needed for a lossless
// round-trip that the editor shouldn't have to think about (asset tokens,
// material params, mixamo rig pose tree) rides along in `source`.
//
// Gray-box mode: textures are not represented. Entities carry an optional
// flat `color`; anything without one renders flat-gray shaded. An LLM is
// free to add primitives and recolor without ever touching texture data.
import { ObjectJSON } from "../proxy/storyteller_proxy_3d_object";
// Bumped when the descriptor shape changes in a non-backward-compatible
// way. `applySceneDescriptor` reads it to decide how to interpret input.
export const SCENE_DESCRIPTOR_VERSION = 1;
// Default flat-gray for entities that don't specify a color (gray-box).
export const GRAY_BOX_COLOR = "#9a9a9a";
export interface Vec3 {
x: number;
y: number;
z: number;
}
// What kind of thing an entity is. Drives how `applySceneDescriptor`
// reconstructs it when no `source` is present (e.g. an LLM-authored
// addition). Only `primitive` can be synthesized from scratch today —
// models/characters need their original asset token, carried in `source`.
export type DescriptorEntityKind =
| "primitive"
| "model"
| "character"
| "light"
| "image"
| "point"
| "unknown";
export interface DescriptorTransform {
// World-space, Y-up, right-handed (three.js convention).
position: Vec3;
// Euler XYZ in DEGREES — friendlier for hand/LLM editing than radians.
rotationDeg: Vec3;
scale: Vec3;
}
// One object in the scene. The top-level fields are the editable surface;
// `source` is the lossless passthrough an editor shouldn't touch.
export interface DescriptorEntity {
// Stable identity across a round-trip (the object's uuid). Keep it on
// edits so the apply step updates-in-place; omit it on a brand-new
// entity and the apply step mints a fresh id.
id: string;
name: string;
kind: DescriptorEntityKind;
// Primitive geometry key ("Box", "Sphere", "Cone", "Cylinder",
// "Donut", "PointLight", "Water"). Only meaningful for `primitive`.
shape?: string;
// Flat hex color, e.g. "#9a9a9a". Absent ⇒ gray-box default.
color?: string;
visible?: boolean;
transform: DescriptorTransform;
// True when this entity carries a mixamo rig pose (in `source.rigData`).
// Per-bone pose editing through the descriptor is a follow-up; for now
// the pose round-trips losslessly via `source` and this flag advertises
// its presence to a reader.
hasPose?: boolean;
// Lossless round-trip passthrough. NOT intended for hand/LLM editing.
// When an entity omits it (a new primitive), `applySceneDescriptor`
// synthesizes the underlying object from the editable fields above.
source?: ObjectJSON;
}
// Read-only spatial context for whoever edits the descriptor. Not applied
// back to the scene (camera framing is out of scope for milestone 1).
export interface DescriptorCamera {
position: Vec3;
rotationDeg: Vec3;
}
export interface DescriptorEnvironment {
// Skybox media token, preserved verbatim across a round-trip.
skybox?: string;
}
export interface SceneDescriptor {
descriptorVersion: number;
// Human/LLM-facing note describing conventions, regenerated on export.
units: "meters";
coordinateSystem: "y-up-right-handed";
environment: DescriptorEnvironment;
camera?: DescriptorCamera;
entities: DescriptorEntity[];
}
@@ -0,0 +1,69 @@
// Universal-format export of the live scene — the interchange "fallback"
// alongside the compact JSON descriptor. glTF/GLB is the portable target
// every DCC tool and most LLMs understand; USDZ is offered as a second
// universal option. Both operate on a pruned clone of the scene that
// contains only real content (no grid, gizmo, or internal bbox helpers).
import * as THREE from "three";
import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js";
import { USDZExporter } from "three/addons/exporters/USDZExporter.js";
import type Editor from "../engine/editor";
import { isInternalBbox } from "../engine/internalBbox";
// Collect exportable content into a fresh group. We clone so the export
// pass can't disturb the live scene graph, and we keep only nodes the
// editor considers real objects (those tagged with a media_id), dropping
// editor scaffolding (grid, TransformControls, bbox helpers).
function buildExportGroup(editor: Editor): THREE.Group {
const group = new THREE.Group();
group.name = "ArtcraftSceneExport";
const scene = editor.activeScene?.scene;
if (!scene) return group;
for (const child of scene.children) {
if (isInternalBbox(child)) continue;
if (child.userData?.["media_id"] == null) continue;
group.add(child.clone(true));
}
return group;
}
// glTF (or binary GLB). Returns a Blob ready to download.
export async function exportSceneToGltf(
editor: Editor,
options: { binary?: boolean } = {},
): Promise<Blob> {
const binary = options.binary ?? true;
const group = buildExportGroup(editor);
const exporter = new GLTFExporter();
const result = await new Promise<ArrayBuffer | object>((resolve, reject) => {
exporter.parse(
group,
(out) => resolve(out as ArrayBuffer | object),
(err) => reject(err),
{ binary, onlyVisible: false },
);
});
if (binary) {
return new Blob([result as ArrayBuffer], { type: "model/gltf-binary" });
}
return new Blob([JSON.stringify(result)], { type: "model/gltf+json" });
}
// USDZ. Returns a Blob ready to download.
//
// three 0.171 exposes `parseAsync(scene, options): Promise<Uint8Array>`,
// but the installed @types/three only declares the older callback-shaped
// `parse`. Reach the runtime method through a narrow typed view so this
// is correct at runtime and still type-checks.
type UsdzAsync = {
parseAsync(scene: THREE.Object3D, options?: object): Promise<Uint8Array>;
};
export async function exportSceneToUsdz(editor: Editor): Promise<Blob> {
const group = buildExportGroup(editor);
const exporter = new USDZExporter() as unknown as UsdzAsync;
const bytes = await exporter.parseAsync(group);
return new Blob([bytes], { type: "model/vnd.usdz+zip" });
}
@@ -2,6 +2,15 @@ import { create } from "zustand";
const ENABLED_STORAGE_KEY = "artcraft_experimental_enabled";
const STORYBOARD_STORAGE_KEY = "artcraft_experimental_storyboard_page";
// Read directly by the pagescene lib (it can't import this store without
// inverting lib layering), so the key string is a cross-lib contract.
const SCENE_ENHANCEMENT_STORAGE_KEY =
"artcraft_experimental_scene_enhancement";
// Same-tab broadcast so other libs that read these flags off localStorage
// can refresh reactively (the native `storage` event only fires in *other*
// tabs). Listeners: see useExperimentalFlag in @storyteller/ui-pagescene.
export const EXPERIMENTAL_FLAGS_CHANGED_EVENT = "artcraft_experimental_changed";
const readBoolFlag = (key: string): boolean => {
if (typeof window === "undefined") return false;
@@ -23,19 +32,31 @@ const writeBoolFlag = (key: string, enabled: boolean) => {
} catch {
// ignore storage failures
}
try {
window.dispatchEvent(
new CustomEvent(EXPERIMENTAL_FLAGS_CHANGED_EVENT, {
detail: { key, enabled },
}),
);
} catch {
// ignore dispatch failures
}
};
interface ExperimentalState {
enabled: boolean;
storyboardPageEnabled: boolean;
sceneEnhancementEnabled: boolean;
enable: () => void;
disable: () => void;
setStoryboardPageEnabled: (enabled: boolean) => void;
setSceneEnhancementEnabled: (enabled: boolean) => void;
}
export const useExperimentalStore = create<ExperimentalState>((set) => ({
enabled: readBoolFlag(ENABLED_STORAGE_KEY),
storyboardPageEnabled: readBoolFlag(STORYBOARD_STORAGE_KEY),
sceneEnhancementEnabled: readBoolFlag(SCENE_ENHANCEMENT_STORAGE_KEY),
enable: () => {
writeBoolFlag(ENABLED_STORAGE_KEY, true);
set({ enabled: true });
@@ -44,13 +65,25 @@ export const useExperimentalStore = create<ExperimentalState>((set) => ({
// Resetting experimental clears every gated feature flag too.
writeBoolFlag(ENABLED_STORAGE_KEY, false);
writeBoolFlag(STORYBOARD_STORAGE_KEY, false);
set({ enabled: false, storyboardPageEnabled: false });
writeBoolFlag(SCENE_ENHANCEMENT_STORAGE_KEY, false);
set({
enabled: false,
storyboardPageEnabled: false,
sceneEnhancementEnabled: false,
});
},
setStoryboardPageEnabled: (enabled: boolean) => {
writeBoolFlag(STORYBOARD_STORAGE_KEY, enabled);
set({ storyboardPageEnabled: enabled });
},
setSceneEnhancementEnabled: (enabled: boolean) => {
writeBoolFlag(SCENE_ENHANCEMENT_STORAGE_KEY, enabled);
set({ sceneEnhancementEnabled: enabled });
},
}));
export const useStoryboardPageEnabled = () =>
useExperimentalStore((s) => s.enabled && s.storyboardPageEnabled);
export const useSceneEnhancementEnabled = () =>
useExperimentalStore((s) => s.enabled && s.sceneEnhancementEnabled);
@@ -18,6 +18,12 @@ export const ExperimentalSettingsPane = ({ onStoryboardPageDisable }: Props) =>
const setStoryboardPageEnabled = useExperimentalStore(
(s) => s.setStoryboardPageEnabled,
);
const sceneEnhancementEnabled = useExperimentalStore(
(s) => s.sceneEnhancementEnabled,
);
const setSceneEnhancementEnabled = useExperimentalStore(
(s) => s.setSceneEnhancementEnabled,
);
const [isDisableConfirmOpen, setIsDisableConfirmOpen] = useState(false);
const fireToggleEvent = (enabled: boolean) => {
@@ -27,6 +33,14 @@ export const ExperimentalSettingsPane = ({ onStoryboardPageDisable }: Props) =>
});
};
const handleSceneEnhancementToggle = (enabled: boolean) => {
setSceneEnhancementEnabled(enabled);
gtagEvent("toggle_experimental_feature", {
feature: "scene_enhancement",
enabled: String(enabled),
});
};
const handleStoryboardToggle = (enabled: boolean) => {
if (enabled) {
setStoryboardPageEnabled(true);
@@ -59,6 +73,23 @@ export const ExperimentalSettingsPane = ({ onStoryboardPageDisable }: Props) =>
setEnabled={handleStoryboardToggle}
/>
</div>
<div className="flex items-start justify-between gap-4">
<div className="flex flex-col gap-0.5">
<Label htmlFor="experimental-scene-enhancement">
Scene enhancement
</Label>
<p className="text-xs opacity-70">
Adds an Enhance panel to the 3D editor for exporting and
re-importing a universal scene descriptor (and glTF/USDZ).
In-development; expect rough edges.
</p>
</div>
<Switch
enabled={sceneEnhancementEnabled}
setEnabled={handleSceneEnhancementToggle}
/>
</div>
</div>
<ExperimentalConfirmModal
isOpen={isDisableConfirmOpen}