mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-16 06:42:14 +00:00
Add experimental scene-descriptor enhancement (manual round-trip)
Menu-gated behind a new experimental flag (artcraft_experimental_scene_enhancement). Adds a universal scene descriptor: a compact, LLM-friendly projection of the PageScene that can be exported, edited, and applied back. - scene_descriptor module: versioned descriptor schema, export (scene -> descriptor), apply (descriptor -> scene via existing load path), and glTF/USDZ universal export. - Gray-box: no texture data; entities carry a flat color, default flat-gray shaded. - Mixamo rig poses round-trip losslessly via the per-entity source passthrough. - Enhance panel in the 3D editor (export/copy/download/apply + glTF/USDZ). - experimental-store: new sceneEnhancementEnabled flag, broadcast on change so the pagescene lib reacts via useExperimentalFlag without an inverted lib dependency. Milestone 1 is a manual round-trip; in-app LLM call and editable bone poses deferred.
This commit is contained in:
1 parent
4d5f8e9c21
commit
4cb498111d
12 files changed
+874
-1
No files matched your search
@@ -181,6 +181,32 @@ export { default as dragAndDrop } from "./lib/DragAndDrop/DndAsset";
|
||||
// Scene-generation metadata helper (used by save flow + cache snapshot).
|
||||
export { getSceneGenerationMetaData } from "./lib/sceneMetadata";
|
||||
|
||||
// Experimental "scene enhancement" — universal scene descriptor round-trip
|
||||
// (export → edit/LLM → apply) plus glTF/USDZ universal export.
|
||||
export {
|
||||
SCENE_DESCRIPTOR_VERSION,
|
||||
GRAY_BOX_COLOR,
|
||||
buildSceneDescriptor,
|
||||
applySceneDescriptor,
|
||||
exportSceneToGltf,
|
||||
exportSceneToUsdz,
|
||||
} from "./lib/scene_descriptor";
|
||||
export type {
|
||||
SceneDescriptor,
|
||||
DescriptorEntity,
|
||||
DescriptorEntityKind,
|
||||
DescriptorTransform,
|
||||
DescriptorCamera,
|
||||
DescriptorEnvironment,
|
||||
ApplyResult,
|
||||
} from "./lib/scene_descriptor";
|
||||
export { SceneDescriptorPanel } from "./lib/comps/SceneDescriptorPanel";
|
||||
export {
|
||||
useExperimentalFlag,
|
||||
useSceneEnhancementFlag,
|
||||
SCENE_ENHANCEMENT_FLAG_KEY,
|
||||
} from "./lib/hooks/useExperimentalFlag";
|
||||
|
||||
// React UI components. Viewport-dependent comps read viewport size
|
||||
// through useViewportSize, which falls back to window.innerWidth/
|
||||
// innerHeight when no host adapter is available.
|
||||
|
||||
@@ -52,6 +52,7 @@ import { FocalLengthDisplay } from "./comps/FocalLengthDisplay/FocalLengthDispla
|
||||
import { OnboardingHelper } from "./comps/OnboardingHelper";
|
||||
import { PerfStatsOverlay } from "./comps/PerfStatsOverlay";
|
||||
import { EntranceDebugPanel } from "./comps/EntranceDebugPanel";
|
||||
import { SceneDescriptorPanel } from "./comps/SceneDescriptorPanel";
|
||||
import { Outliner } from "./comps/Outliner";
|
||||
import { PoseModeSelector } from "./comps/PoseModeSelector";
|
||||
import { PreviewBox } from "./comps/PreviewBox";
|
||||
@@ -63,6 +64,7 @@ import { useFreeCam } from "./hooks/useFreeCam";
|
||||
import { useViewportPointer } from "./hooks/useViewportPointer";
|
||||
import { useViewportKeyboard } from "./hooks/useViewportKeyboard";
|
||||
import { useViewportSize } from "./hooks/useViewportSize";
|
||||
import { useSceneEnhancementFlag } from "./hooks/useExperimentalFlag";
|
||||
import { GridVisibleChangedEvent } from "./engine/events/EngineEvent";
|
||||
import { pickDropPosition } from "./engine/pickDropPosition";
|
||||
import { AssetType, CameraAspectRatio } from "./enums";
|
||||
@@ -212,6 +214,7 @@ export const Stage3DBody = ({
|
||||
);
|
||||
|
||||
const editor = useContext(EngineContext);
|
||||
const sceneEnhancementEnabled = useSceneEnhancementFlag();
|
||||
|
||||
// Reactive viewport sizing. useViewportSize listens to window
|
||||
// resize and re-renders the component. Falls back to
|
||||
@@ -435,6 +438,7 @@ export const Stage3DBody = ({
|
||||
|
||||
<PerfStatsOverlay />
|
||||
{import.meta.env.DEV && <EntranceDebugPanel />}
|
||||
{sceneEnhancementEnabled && <SceneDescriptorPanel />}
|
||||
<FocalLengthDisplay />
|
||||
<PoseModeSelector />
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
// Experimental "scene enhancement" panel. Floating control in the 3D
|
||||
// editor (gated behind the experimental flag) that round-trips the scene
|
||||
// through a universal descriptor: export the current scene to editable
|
||||
// JSON, edit it (by hand or via an external LLM), paste it back, and apply.
|
||||
// Also offers glTF/USDZ export as a universal interchange fallback.
|
||||
//
|
||||
// Milestone 1 is a manual round-trip — no in-app LLM call yet. The Apply
|
||||
// step rebuilds the scene from the pasted descriptor.
|
||||
|
||||
import { useContext, useState } from "react";
|
||||
import { EngineContext } from "../contexts/EngineContext/EngineContext";
|
||||
import {
|
||||
applySceneDescriptor,
|
||||
buildSceneDescriptor,
|
||||
exportSceneToGltf,
|
||||
exportSceneToUsdz,
|
||||
SceneDescriptor,
|
||||
} from "../scene_descriptor";
|
||||
|
||||
type Status = { kind: "idle" | "ok" | "err"; message: string };
|
||||
|
||||
const IDLE: Status = { kind: "idle", message: "" };
|
||||
|
||||
export function SceneDescriptorPanel() {
|
||||
const editor = useContext(EngineContext);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [text, setText] = useState("");
|
||||
const [status, setStatus] = useState<Status>(IDLE);
|
||||
const [busy, setBusy] = useState(false);
|
||||
|
||||
const handleExport = () => {
|
||||
if (!editor) return;
|
||||
try {
|
||||
const descriptor = buildSceneDescriptor(editor);
|
||||
setText(JSON.stringify(descriptor, null, 2));
|
||||
setStatus({
|
||||
kind: "ok",
|
||||
message: `Exported ${descriptor.entities.length} entit${
|
||||
descriptor.entities.length === 1 ? "y" : "ies"
|
||||
}.`,
|
||||
});
|
||||
} catch (err) {
|
||||
setStatus({ kind: "err", message: errMsg("Export failed", err) });
|
||||
}
|
||||
};
|
||||
|
||||
const handleCopy = async () => {
|
||||
if (!text) return;
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
setStatus({ kind: "ok", message: "Copied descriptor to clipboard." });
|
||||
} catch (err) {
|
||||
setStatus({ kind: "err", message: errMsg("Copy failed", err) });
|
||||
}
|
||||
};
|
||||
|
||||
const handleDownloadJson = () => {
|
||||
if (!text) return;
|
||||
downloadBlob(
|
||||
new Blob([text], { type: "application/json" }),
|
||||
"scene-descriptor.json",
|
||||
);
|
||||
};
|
||||
|
||||
const handleDownloadGltf = async () => {
|
||||
if (!editor) return;
|
||||
await withBusy(async () => {
|
||||
const blob = await exportSceneToGltf(editor, { binary: true });
|
||||
downloadBlob(blob, "scene.glb");
|
||||
setStatus({ kind: "ok", message: "Exported scene.glb." });
|
||||
}, "glTF export failed");
|
||||
};
|
||||
|
||||
const handleDownloadUsdz = async () => {
|
||||
if (!editor) return;
|
||||
await withBusy(async () => {
|
||||
const blob = await exportSceneToUsdz(editor);
|
||||
downloadBlob(blob, "scene.usdz");
|
||||
setStatus({ kind: "ok", message: "Exported scene.usdz." });
|
||||
}, "USDZ export failed");
|
||||
};
|
||||
|
||||
const handleApply = async () => {
|
||||
if (!editor || !text.trim()) return;
|
||||
let descriptor: SceneDescriptor;
|
||||
try {
|
||||
descriptor = JSON.parse(text) as SceneDescriptor;
|
||||
} catch (err) {
|
||||
setStatus({ kind: "err", message: errMsg("Invalid JSON", err) });
|
||||
return;
|
||||
}
|
||||
await withBusy(async () => {
|
||||
const result = await applySceneDescriptor(editor, descriptor);
|
||||
const skippedNote = result.skipped
|
||||
? ` (${result.skipped} skipped — new models/characters need a source)`
|
||||
: "";
|
||||
setStatus({
|
||||
kind: "ok",
|
||||
message: `Applied ${result.applied} entit${
|
||||
result.applied === 1 ? "y" : "ies"
|
||||
}${skippedNote}.`,
|
||||
});
|
||||
}, "Apply failed");
|
||||
};
|
||||
|
||||
const withBusy = async (fn: () => Promise<void>, errLabel: string) => {
|
||||
setBusy(true);
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
setStatus({ kind: "err", message: errMsg(errLabel, err) });
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (!open) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen(true)}
|
||||
className="pointer-events-auto absolute right-2 top-16 z-50 rounded-lg border border-white/10 bg-black/70 px-3 py-1.5 text-xs font-medium text-white/80 backdrop-blur-md transition-colors hover:bg-white/10 hover:text-white/90"
|
||||
>
|
||||
✦ Enhance
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="pointer-events-auto absolute right-2 top-16 z-50 flex w-72 flex-col gap-2 rounded-lg border border-white/10 bg-black/70 p-3 text-xs text-white/80 backdrop-blur-md">
|
||||
<div className="flex items-center justify-between font-medium text-white/90">
|
||||
<span>Scene enhancement</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen(false)}
|
||||
className="rounded px-1 text-white/50 hover:text-white/90"
|
||||
aria-label="Close"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-1.5">
|
||||
<PanelButton onClick={handleExport} disabled={!editor || busy}>
|
||||
Export descriptor
|
||||
</PanelButton>
|
||||
<PanelButton onClick={handleCopy} disabled={!text || busy}>
|
||||
Copy
|
||||
</PanelButton>
|
||||
<PanelButton onClick={handleDownloadJson} disabled={!text || busy}>
|
||||
Download .json
|
||||
</PanelButton>
|
||||
<PanelButton onClick={handleApply} disabled={!editor || !text || busy}>
|
||||
Apply to scene
|
||||
</PanelButton>
|
||||
<PanelButton onClick={handleDownloadGltf} disabled={!editor || busy}>
|
||||
glTF (.glb)
|
||||
</PanelButton>
|
||||
<PanelButton onClick={handleDownloadUsdz} disabled={!editor || busy}>
|
||||
USDZ
|
||||
</PanelButton>
|
||||
</div>
|
||||
|
||||
<textarea
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
spellCheck={false}
|
||||
placeholder="Export a descriptor, edit it (or paste an LLM-edited one), then Apply to scene."
|
||||
className="h-40 w-full resize-y rounded border border-white/10 bg-black/50 p-2 font-mono text-[10px] leading-snug text-white/80 outline-none focus:border-white/30"
|
||||
/>
|
||||
|
||||
{status.kind !== "idle" && (
|
||||
<p
|
||||
className={
|
||||
status.kind === "err" ? "text-red-300" : "text-emerald-300"
|
||||
}
|
||||
>
|
||||
{status.message}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function PanelButton({
|
||||
onClick,
|
||||
disabled,
|
||||
children,
|
||||
}: {
|
||||
onClick: () => void;
|
||||
disabled?: boolean;
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClick}
|
||||
disabled={disabled}
|
||||
className="rounded border border-white/10 py-1 text-white/70 transition-colors hover:bg-white/10 hover:text-white/90 disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function downloadBlob(blob: Blob, filename: string) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement("a");
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
// Revoke on the next tick so the click-driven navigation has started.
|
||||
setTimeout(() => URL.revokeObjectURL(url), 0);
|
||||
}
|
||||
|
||||
function errMsg(label: string, err: unknown): string {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
return `${label}: ${detail}`;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Read an experimental feature flag without importing the settings-modal
|
||||
// store (that lib sits above pagescene in the layering, so we can't depend
|
||||
// on it). The flag's source of truth is localStorage; the settings store
|
||||
// broadcasts an `artcraft_experimental_changed` CustomEvent on every
|
||||
// toggle so we can refresh same-tab. The cross-tab `storage` event covers
|
||||
// other tabs. The string keys + event name are the shared contract with
|
||||
// experimental-store.ts.
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
const MASTER_KEY = "artcraft_experimental_enabled";
|
||||
const FLAGS_CHANGED_EVENT = "artcraft_experimental_changed";
|
||||
|
||||
const read = (featureKey: string): boolean => {
|
||||
if (typeof window === "undefined") return false;
|
||||
try {
|
||||
return (
|
||||
window.localStorage.getItem(MASTER_KEY) === "true" &&
|
||||
window.localStorage.getItem(featureKey) === "true"
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export function useExperimentalFlag(featureKey: string): boolean {
|
||||
const [enabled, setEnabled] = useState(() => read(featureKey));
|
||||
|
||||
useEffect(() => {
|
||||
const refresh = () => setEnabled(read(featureKey));
|
||||
refresh();
|
||||
window.addEventListener(FLAGS_CHANGED_EVENT, refresh);
|
||||
window.addEventListener("storage", refresh);
|
||||
return () => {
|
||||
window.removeEventListener(FLAGS_CHANGED_EVENT, refresh);
|
||||
window.removeEventListener("storage", refresh);
|
||||
};
|
||||
}, [featureKey]);
|
||||
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// The scene-enhancement feature gate.
|
||||
export const SCENE_ENHANCEMENT_FLAG_KEY =
|
||||
"artcraft_experimental_scene_enhancement";
|
||||
|
||||
export const useSceneEnhancementFlag = () =>
|
||||
useExperimentalFlag(SCENE_ENHANCEMENT_FLAG_KEY);
|
||||
@@ -0,0 +1,61 @@
|
||||
# Scene Descriptor (experimental "scene enhancement")
|
||||
|
||||
A compact, LLM-friendly projection of the live PageScene that can be
|
||||
exported, edited, and applied back — the foundation for using an LLM to
|
||||
iterate the contents of a 3D scene.
|
||||
|
||||
This is **menu-gated** behind the experimental flag
|
||||
`artcraft_experimental_scene_enhancement` (Settings → Experimental →
|
||||
"Scene enhancement"). When on, an **✦ Enhance** panel appears in the 3D
|
||||
editor.
|
||||
|
||||
## Round-trip
|
||||
|
||||
```
|
||||
buildSceneDescriptor(editor) → SceneDescriptor (JSON)
|
||||
↓ edit by hand, or paste through any external LLM
|
||||
applySceneDescriptor(editor, descriptor) → scene rebuilt
|
||||
```
|
||||
|
||||
Milestone 1 is a **manual** round-trip — no in-app LLM call yet. Export the
|
||||
descriptor, edit/enhance it however you like, paste it back, Apply.
|
||||
|
||||
## Format
|
||||
|
||||
`SceneDescriptor` (see `scene_descriptor.ts`) is a lossy, editable *view*
|
||||
of the scene:
|
||||
|
||||
- `entities[]` — one per scene object with `id`, `name`, `kind`
|
||||
(`primitive | model | character | light | image | point`), optional
|
||||
`shape`/`color`/`visible`, and a `transform` (`position`, `rotationDeg`
|
||||
in **degrees**, `scale`).
|
||||
- `environment.skybox`, plus a read-only `camera` for spatial context.
|
||||
|
||||
Each entity also carries `source`: the full internal `ObjectJSON`. This is
|
||||
**not** for hand-editing — it's the lossless passthrough (asset token,
|
||||
material, mixamo rig pose) that lets a round-trip reconstruct an object
|
||||
exactly. An entity with no `source` is treated as new; only **primitives**
|
||||
can be synthesized from scratch (models/characters need their asset token).
|
||||
|
||||
### Gray-box
|
||||
|
||||
Textures are intentionally not represented. Entities carry a flat `color`
|
||||
only; anything without one renders flat-gray (`GRAY_BOX_COLOR`). An LLM can
|
||||
add and recolor primitives without ever touching texture data.
|
||||
|
||||
## Universal export (fallback)
|
||||
|
||||
`exportSceneToGltf` / `exportSceneToUsdz` export the scene to glTF/GLB and
|
||||
USDZ — the portable interchange formats, alongside the JSON descriptor.
|
||||
|
||||
## Roadmap / not done yet
|
||||
|
||||
- **Editable mixamo pose** — bone poses currently round-trip losslessly via
|
||||
`source.rigData` but are not yet decoded into a per-bone editable surface
|
||||
(`hasPose` advertises their presence). Decode/encode is the next step so
|
||||
an LLM can re-pose characters.
|
||||
- **In-app LLM call** — placement (backend `storyteller_web` endpoint vs.
|
||||
frontend/Tauri) is deliberately deferred; milestone 1 is manual.
|
||||
- **Non-destructive apply** — Apply currently rebuilds the scene and clears
|
||||
the undo stack (same path as "Reset to original"). A diff-based apply
|
||||
(move/add/remove in place, undoable) is a follow-up.
|
||||
@@ -0,0 +1,149 @@
|
||||
// Apply: SceneDescriptor → live PageScene.
|
||||
//
|
||||
// We translate the descriptor back into the ObjectJSON[] the editor's
|
||||
// existing load path already understands, then hand it to
|
||||
// editor.applyJson (which clears the undo stack and rebuilds the scene).
|
||||
// Existing entities are reconstructed from their lossless `source`;
|
||||
// entities the editor never saw (e.g. an LLM-authored primitive) are
|
||||
// synthesized from the editable fields.
|
||||
|
||||
import * as THREE from "three";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type Editor from "../engine/editor";
|
||||
import { ObjectJSON } from "../proxy/storyteller_proxy_3d_object";
|
||||
import { MediaFileType } from "../enums";
|
||||
import {
|
||||
DescriptorEntity,
|
||||
GRAY_BOX_COLOR,
|
||||
SceneDescriptor,
|
||||
Vec3,
|
||||
} from "./scene_descriptor";
|
||||
|
||||
export interface ApplyResult {
|
||||
applied: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export async function applySceneDescriptor(
|
||||
editor: Editor,
|
||||
descriptor: SceneDescriptor,
|
||||
): Promise<ApplyResult> {
|
||||
const entities = descriptor?.entities ?? [];
|
||||
let skipped = 0;
|
||||
|
||||
const sceneJson: ObjectJSON[] = [];
|
||||
for (const entity of entities) {
|
||||
const obj = entityToObjectJson(entity, editor.version);
|
||||
if (obj) {
|
||||
sceneJson.push(obj);
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
const wrapper = {
|
||||
version: editor.version,
|
||||
scene: sceneJson,
|
||||
skybox: descriptor?.environment?.skybox ?? editor.activeScene.skybox ?? "",
|
||||
};
|
||||
|
||||
await editor.applyJson(JSON.stringify(wrapper));
|
||||
return { applied: sceneJson.length, skipped };
|
||||
}
|
||||
|
||||
// Build an ObjectJSON for a single entity. Returns undefined when the
|
||||
// entity can't be reconstructed (a new model/character with no `source`,
|
||||
// since the asset token is unknown).
|
||||
function entityToObjectJson(
|
||||
entity: DescriptorEntity,
|
||||
version: number,
|
||||
): ObjectJSON | undefined {
|
||||
const base = entity.source
|
||||
? cloneObjectJson(entity.source)
|
||||
: synthesize(entity, version);
|
||||
if (!base) return undefined;
|
||||
|
||||
// Overlay the editable surface onto the (possibly preserved) base.
|
||||
base.object_uuid = entity.id || base.object_uuid || uuidv4();
|
||||
if (entity.name) {
|
||||
base.object_user_data_name = entity.name;
|
||||
base.user_data = { ...base.user_data, name: entity.name };
|
||||
}
|
||||
if (entity.color) base.color = entity.color;
|
||||
if (entity.visible !== undefined) base.visible = entity.visible;
|
||||
|
||||
const t = entity.transform;
|
||||
if (t) {
|
||||
base.position = vec(t.position);
|
||||
base.rotation = eulerDegToRad(t.rotationDeg);
|
||||
base.scale = vec(t.scale, 1);
|
||||
}
|
||||
|
||||
// A primitive whose shape changed needs the geometry key updated so the
|
||||
// loader's instantiate() picks the new geometry on rebuild.
|
||||
if (base.media_file_token === "Parim" && entity.shape) {
|
||||
base.object_name = entity.shape;
|
||||
base.user_data = {
|
||||
...base.user_data,
|
||||
shapeKey: entity.shape,
|
||||
shapeType: entity.shape,
|
||||
isShape: true,
|
||||
};
|
||||
}
|
||||
|
||||
return base;
|
||||
}
|
||||
|
||||
// Synthesize an ObjectJSON for an entity that has no `source`. Only
|
||||
// primitives are supported — models/characters require an asset token we
|
||||
// can't invent.
|
||||
function synthesize(
|
||||
entity: DescriptorEntity,
|
||||
version: number,
|
||||
): ObjectJSON | undefined {
|
||||
if (entity.kind !== "primitive" || !entity.shape) return undefined;
|
||||
const shape = entity.shape;
|
||||
return {
|
||||
version,
|
||||
position: { x: 0, y: 0, z: 0 },
|
||||
rotation: { x: 0, y: 0, z: 0 },
|
||||
scale: { x: 1, y: 1, z: 1 },
|
||||
object_name: shape,
|
||||
object_uuid: entity.id || uuidv4(),
|
||||
object_user_data_name: entity.name || shape,
|
||||
media_file_token: "Parim",
|
||||
color: entity.color || GRAY_BOX_COLOR,
|
||||
metalness: 0,
|
||||
shininess: 0.5,
|
||||
specular: 0.5,
|
||||
locked: false,
|
||||
visible: entity.visible ?? true,
|
||||
rigData: undefined,
|
||||
user_data: {
|
||||
name: entity.name || shape,
|
||||
isShape: true,
|
||||
shapeKey: shape,
|
||||
shapeType: shape,
|
||||
media_file_type: MediaFileType.None,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Deep-clone so we never mutate the descriptor the caller still holds.
|
||||
function cloneObjectJson(obj: ObjectJSON): ObjectJSON {
|
||||
return JSON.parse(JSON.stringify(obj));
|
||||
}
|
||||
|
||||
function vec(v: Vec3 | undefined, fallback = 0): { x: number; y: number; z: number } {
|
||||
if (!v) return { x: fallback, y: fallback, z: fallback };
|
||||
return { x: v.x ?? fallback, y: v.y ?? fallback, z: v.z ?? fallback };
|
||||
}
|
||||
|
||||
function eulerDegToRad(v: Vec3 | undefined): { x: number; y: number; z: number } {
|
||||
if (!v) return { x: 0, y: 0, z: 0 };
|
||||
return {
|
||||
x: THREE.MathUtils.degToRad(v.x ?? 0),
|
||||
y: THREE.MathUtils.degToRad(v.y ?? 0),
|
||||
z: THREE.MathUtils.degToRad(v.z ?? 0),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Export: live PageScene → SceneDescriptor.
|
||||
//
|
||||
// We lean on the existing serialization machinery (StoryTellerProxyScene)
|
||||
// to snapshot the scene into ObjectJSON[], then project each object into
|
||||
// the compact, editable descriptor entity while stashing the full
|
||||
// ObjectJSON in `source` for a lossless round-trip.
|
||||
|
||||
import * as THREE from "three";
|
||||
import type Editor from "../engine/editor";
|
||||
import { StoryTellerProxyScene } from "../proxy/storyteller_proxy_scene";
|
||||
import { ObjectJSON } from "../proxy/storyteller_proxy_3d_object";
|
||||
import {
|
||||
DescriptorCamera,
|
||||
DescriptorEntity,
|
||||
DescriptorEntityKind,
|
||||
SceneDescriptor,
|
||||
SCENE_DESCRIPTOR_VERSION,
|
||||
Vec3,
|
||||
} from "./scene_descriptor";
|
||||
|
||||
export function buildSceneDescriptor(editor: Editor): SceneDescriptor {
|
||||
const scene = editor.activeScene;
|
||||
const proxy = new StoryTellerProxyScene(editor.version, scene);
|
||||
const objects = proxy.saveToScene(editor.version);
|
||||
|
||||
const entities = objects.map(objectJsonToEntity);
|
||||
|
||||
return {
|
||||
descriptorVersion: SCENE_DESCRIPTOR_VERSION,
|
||||
units: "meters",
|
||||
coordinateSystem: "y-up-right-handed",
|
||||
environment: { skybox: scene.skybox || undefined },
|
||||
camera: readCamera(editor),
|
||||
entities,
|
||||
};
|
||||
}
|
||||
|
||||
function objectJsonToEntity(obj: ObjectJSON): DescriptorEntity {
|
||||
const kind = classifyKind(obj);
|
||||
const entity: DescriptorEntity = {
|
||||
id: obj.object_uuid,
|
||||
name: obj.object_user_data_name || obj.object_name || "",
|
||||
kind,
|
||||
color: obj.color || undefined,
|
||||
visible: obj.visible !== false,
|
||||
transform: {
|
||||
position: radSafeVec(obj.position),
|
||||
rotationDeg: eulerRadToDeg(obj.rotation),
|
||||
scale: radSafeVec(obj.scale),
|
||||
},
|
||||
source: obj,
|
||||
};
|
||||
|
||||
if (kind === "primitive") {
|
||||
entity.shape =
|
||||
(obj.user_data?.shapeType as string | undefined) ??
|
||||
(obj.user_data?.shapeKey as string | undefined) ??
|
||||
obj.object_name;
|
||||
}
|
||||
if (obj.rigData) {
|
||||
entity.hasPose = true;
|
||||
}
|
||||
return entity;
|
||||
}
|
||||
|
||||
// Mirror of the token dispatch in StoryTellerProxyScene.loadFromSceneJson:
|
||||
// the media_file_token is the discriminator the loader switches on.
|
||||
function classifyKind(obj: ObjectJSON): DescriptorEntityKind {
|
||||
const token = obj.media_file_token ?? "";
|
||||
if (token === "Parim" || obj.user_data?.isShape === true) return "primitive";
|
||||
if (token === "DirectionalLight") return "light";
|
||||
if (token.includes("Point::")) return "point";
|
||||
if (token.includes("Image::")) return "image";
|
||||
if (token.includes("m_")) {
|
||||
return obj.user_data?.isCharacter === true ? "character" : "model";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function readCamera(editor: Editor): DescriptorCamera | undefined {
|
||||
const cam = editor.cameraController?.camera;
|
||||
if (!cam) return undefined;
|
||||
return {
|
||||
position: { x: cam.position.x, y: cam.position.y, z: cam.position.z },
|
||||
rotationDeg: eulerRadToDeg(cam.rotation),
|
||||
};
|
||||
}
|
||||
|
||||
function eulerRadToDeg(r: { x: number; y: number; z: number }): Vec3 {
|
||||
return {
|
||||
x: round(THREE.MathUtils.radToDeg(r.x)),
|
||||
y: round(THREE.MathUtils.radToDeg(r.y)),
|
||||
z: round(THREE.MathUtils.radToDeg(r.z)),
|
||||
};
|
||||
}
|
||||
|
||||
function radSafeVec(v: { x: number; y: number; z: number }): Vec3 {
|
||||
return { x: round(v.x), y: round(v.y), z: round(v.z) };
|
||||
}
|
||||
|
||||
// Trim float noise so the exported JSON reads cleanly. 5 decimals keeps
|
||||
// sub-millimeter precision while dropping 1e-16 dust from THREE math.
|
||||
function round(n: number): number {
|
||||
return Math.round((n + Number.EPSILON) * 1e5) / 1e5;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// Experimental "scene enhancement" — universal scene descriptor.
|
||||
// Round-trip: buildSceneDescriptor → (edit / LLM) → applySceneDescriptor.
|
||||
|
||||
export {
|
||||
SCENE_DESCRIPTOR_VERSION,
|
||||
GRAY_BOX_COLOR,
|
||||
} from "./scene_descriptor";
|
||||
export type {
|
||||
SceneDescriptor,
|
||||
DescriptorEntity,
|
||||
DescriptorEntityKind,
|
||||
DescriptorTransform,
|
||||
DescriptorCamera,
|
||||
DescriptorEnvironment,
|
||||
Vec3,
|
||||
} from "./scene_descriptor";
|
||||
|
||||
export { buildSceneDescriptor } from "./export_descriptor";
|
||||
export { applySceneDescriptor } from "./apply_descriptor";
|
||||
export type { ApplyResult } from "./apply_descriptor";
|
||||
export { exportSceneToGltf, exportSceneToUsdz } from "./universal_export";
|
||||
@@ -0,0 +1,105 @@
|
||||
// Universal Scene Descriptor — a compact, LLM-friendly projection of the
|
||||
// live 3D scene.
|
||||
//
|
||||
// This is the editable interchange format for the experimental
|
||||
// "scene enhancement" feature: we export the PageScene into a descriptor,
|
||||
// let a human (or, later, an LLM) edit it, then apply it back to update
|
||||
// the scene. It is intentionally a *lossy view* of the heavy internal
|
||||
// `ObjectJSON` — only the spatial/semantic fields a model can reason
|
||||
// about are surfaced at the top level. Everything needed for a lossless
|
||||
// round-trip that the editor shouldn't have to think about (asset tokens,
|
||||
// material params, mixamo rig pose tree) rides along in `source`.
|
||||
//
|
||||
// Gray-box mode: textures are not represented. Entities carry an optional
|
||||
// flat `color`; anything without one renders flat-gray shaded. An LLM is
|
||||
// free to add primitives and recolor without ever touching texture data.
|
||||
|
||||
import { ObjectJSON } from "../proxy/storyteller_proxy_3d_object";
|
||||
|
||||
// Bumped when the descriptor shape changes in a non-backward-compatible
|
||||
// way. `applySceneDescriptor` reads it to decide how to interpret input.
|
||||
export const SCENE_DESCRIPTOR_VERSION = 1;
|
||||
|
||||
// Default flat-gray for entities that don't specify a color (gray-box).
|
||||
export const GRAY_BOX_COLOR = "#9a9a9a";
|
||||
|
||||
export interface Vec3 {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
}
|
||||
|
||||
// What kind of thing an entity is. Drives how `applySceneDescriptor`
|
||||
// reconstructs it when no `source` is present (e.g. an LLM-authored
|
||||
// addition). Only `primitive` can be synthesized from scratch today —
|
||||
// models/characters need their original asset token, carried in `source`.
|
||||
export type DescriptorEntityKind =
|
||||
| "primitive"
|
||||
| "model"
|
||||
| "character"
|
||||
| "light"
|
||||
| "image"
|
||||
| "point"
|
||||
| "unknown";
|
||||
|
||||
export interface DescriptorTransform {
|
||||
// World-space, Y-up, right-handed (three.js convention).
|
||||
position: Vec3;
|
||||
// Euler XYZ in DEGREES — friendlier for hand/LLM editing than radians.
|
||||
rotationDeg: Vec3;
|
||||
scale: Vec3;
|
||||
}
|
||||
|
||||
// One object in the scene. The top-level fields are the editable surface;
|
||||
// `source` is the lossless passthrough an editor shouldn't touch.
|
||||
export interface DescriptorEntity {
|
||||
// Stable identity across a round-trip (the object's uuid). Keep it on
|
||||
// edits so the apply step updates-in-place; omit it on a brand-new
|
||||
// entity and the apply step mints a fresh id.
|
||||
id: string;
|
||||
name: string;
|
||||
kind: DescriptorEntityKind;
|
||||
|
||||
// Primitive geometry key ("Box", "Sphere", "Cone", "Cylinder",
|
||||
// "Donut", "PointLight", "Water"). Only meaningful for `primitive`.
|
||||
shape?: string;
|
||||
|
||||
// Flat hex color, e.g. "#9a9a9a". Absent ⇒ gray-box default.
|
||||
color?: string;
|
||||
visible?: boolean;
|
||||
|
||||
transform: DescriptorTransform;
|
||||
|
||||
// True when this entity carries a mixamo rig pose (in `source.rigData`).
|
||||
// Per-bone pose editing through the descriptor is a follow-up; for now
|
||||
// the pose round-trips losslessly via `source` and this flag advertises
|
||||
// its presence to a reader.
|
||||
hasPose?: boolean;
|
||||
|
||||
// Lossless round-trip passthrough. NOT intended for hand/LLM editing.
|
||||
// When an entity omits it (a new primitive), `applySceneDescriptor`
|
||||
// synthesizes the underlying object from the editable fields above.
|
||||
source?: ObjectJSON;
|
||||
}
|
||||
|
||||
// Read-only spatial context for whoever edits the descriptor. Not applied
|
||||
// back to the scene (camera framing is out of scope for milestone 1).
|
||||
export interface DescriptorCamera {
|
||||
position: Vec3;
|
||||
rotationDeg: Vec3;
|
||||
}
|
||||
|
||||
export interface DescriptorEnvironment {
|
||||
// Skybox media token, preserved verbatim across a round-trip.
|
||||
skybox?: string;
|
||||
}
|
||||
|
||||
export interface SceneDescriptor {
|
||||
descriptorVersion: number;
|
||||
// Human/LLM-facing note describing conventions, regenerated on export.
|
||||
units: "meters";
|
||||
coordinateSystem: "y-up-right-handed";
|
||||
environment: DescriptorEnvironment;
|
||||
camera?: DescriptorCamera;
|
||||
entities: DescriptorEntity[];
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Universal-format export of the live scene — the interchange "fallback"
|
||||
// alongside the compact JSON descriptor. glTF/GLB is the portable target
|
||||
// every DCC tool and most LLMs understand; USDZ is offered as a second
|
||||
// universal option. Both operate on a pruned clone of the scene that
|
||||
// contains only real content (no grid, gizmo, or internal bbox helpers).
|
||||
|
||||
import * as THREE from "three";
|
||||
import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js";
|
||||
import { USDZExporter } from "three/addons/exporters/USDZExporter.js";
|
||||
import type Editor from "../engine/editor";
|
||||
import { isInternalBbox } from "../engine/internalBbox";
|
||||
|
||||
// Collect exportable content into a fresh group. We clone so the export
|
||||
// pass can't disturb the live scene graph, and we keep only nodes the
|
||||
// editor considers real objects (those tagged with a media_id), dropping
|
||||
// editor scaffolding (grid, TransformControls, bbox helpers).
|
||||
function buildExportGroup(editor: Editor): THREE.Group {
|
||||
const group = new THREE.Group();
|
||||
group.name = "ArtcraftSceneExport";
|
||||
const scene = editor.activeScene?.scene;
|
||||
if (!scene) return group;
|
||||
for (const child of scene.children) {
|
||||
if (isInternalBbox(child)) continue;
|
||||
if (child.userData?.["media_id"] == null) continue;
|
||||
group.add(child.clone(true));
|
||||
}
|
||||
return group;
|
||||
}
|
||||
|
||||
// glTF (or binary GLB). Returns a Blob ready to download.
|
||||
export async function exportSceneToGltf(
|
||||
editor: Editor,
|
||||
options: { binary?: boolean } = {},
|
||||
): Promise<Blob> {
|
||||
const binary = options.binary ?? true;
|
||||
const group = buildExportGroup(editor);
|
||||
const exporter = new GLTFExporter();
|
||||
|
||||
const result = await new Promise<ArrayBuffer | object>((resolve, reject) => {
|
||||
exporter.parse(
|
||||
group,
|
||||
(out) => resolve(out as ArrayBuffer | object),
|
||||
(err) => reject(err),
|
||||
{ binary, onlyVisible: false },
|
||||
);
|
||||
});
|
||||
|
||||
if (binary) {
|
||||
return new Blob([result as ArrayBuffer], { type: "model/gltf-binary" });
|
||||
}
|
||||
return new Blob([JSON.stringify(result)], { type: "model/gltf+json" });
|
||||
}
|
||||
|
||||
// USDZ. Returns a Blob ready to download.
|
||||
//
|
||||
// three 0.171 exposes `parseAsync(scene, options): Promise<Uint8Array>`,
|
||||
// but the installed @types/three only declares the older callback-shaped
|
||||
// `parse`. Reach the runtime method through a narrow typed view so this
|
||||
// is correct at runtime and still type-checks.
|
||||
type UsdzAsync = {
|
||||
parseAsync(scene: THREE.Object3D, options?: object): Promise<Uint8Array>;
|
||||
};
|
||||
|
||||
export async function exportSceneToUsdz(editor: Editor): Promise<Blob> {
|
||||
const group = buildExportGroup(editor);
|
||||
const exporter = new USDZExporter() as unknown as UsdzAsync;
|
||||
const bytes = await exporter.parseAsync(group);
|
||||
return new Blob([bytes], { type: "model/vnd.usdz+zip" });
|
||||
}
|
||||
@@ -2,6 +2,15 @@ import { create } from "zustand";
|
||||
|
||||
const ENABLED_STORAGE_KEY = "artcraft_experimental_enabled";
|
||||
const STORYBOARD_STORAGE_KEY = "artcraft_experimental_storyboard_page";
|
||||
// Read directly by the pagescene lib (it can't import this store without
|
||||
// inverting lib layering), so the key string is a cross-lib contract.
|
||||
const SCENE_ENHANCEMENT_STORAGE_KEY =
|
||||
"artcraft_experimental_scene_enhancement";
|
||||
|
||||
// Same-tab broadcast so other libs that read these flags off localStorage
|
||||
// can refresh reactively (the native `storage` event only fires in *other*
|
||||
// tabs). Listeners: see useExperimentalFlag in @storyteller/ui-pagescene.
|
||||
export const EXPERIMENTAL_FLAGS_CHANGED_EVENT = "artcraft_experimental_changed";
|
||||
|
||||
const readBoolFlag = (key: string): boolean => {
|
||||
if (typeof window === "undefined") return false;
|
||||
@@ -23,19 +32,31 @@ const writeBoolFlag = (key: string, enabled: boolean) => {
|
||||
} catch {
|
||||
// ignore storage failures
|
||||
}
|
||||
try {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent(EXPERIMENTAL_FLAGS_CHANGED_EVENT, {
|
||||
detail: { key, enabled },
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
// ignore dispatch failures
|
||||
}
|
||||
};
|
||||
|
||||
interface ExperimentalState {
|
||||
enabled: boolean;
|
||||
storyboardPageEnabled: boolean;
|
||||
sceneEnhancementEnabled: boolean;
|
||||
enable: () => void;
|
||||
disable: () => void;
|
||||
setStoryboardPageEnabled: (enabled: boolean) => void;
|
||||
setSceneEnhancementEnabled: (enabled: boolean) => void;
|
||||
}
|
||||
|
||||
export const useExperimentalStore = create<ExperimentalState>((set) => ({
|
||||
enabled: readBoolFlag(ENABLED_STORAGE_KEY),
|
||||
storyboardPageEnabled: readBoolFlag(STORYBOARD_STORAGE_KEY),
|
||||
sceneEnhancementEnabled: readBoolFlag(SCENE_ENHANCEMENT_STORAGE_KEY),
|
||||
enable: () => {
|
||||
writeBoolFlag(ENABLED_STORAGE_KEY, true);
|
||||
set({ enabled: true });
|
||||
@@ -44,13 +65,25 @@ export const useExperimentalStore = create<ExperimentalState>((set) => ({
|
||||
// Resetting experimental clears every gated feature flag too.
|
||||
writeBoolFlag(ENABLED_STORAGE_KEY, false);
|
||||
writeBoolFlag(STORYBOARD_STORAGE_KEY, false);
|
||||
set({ enabled: false, storyboardPageEnabled: false });
|
||||
writeBoolFlag(SCENE_ENHANCEMENT_STORAGE_KEY, false);
|
||||
set({
|
||||
enabled: false,
|
||||
storyboardPageEnabled: false,
|
||||
sceneEnhancementEnabled: false,
|
||||
});
|
||||
},
|
||||
setStoryboardPageEnabled: (enabled: boolean) => {
|
||||
writeBoolFlag(STORYBOARD_STORAGE_KEY, enabled);
|
||||
set({ storyboardPageEnabled: enabled });
|
||||
},
|
||||
setSceneEnhancementEnabled: (enabled: boolean) => {
|
||||
writeBoolFlag(SCENE_ENHANCEMENT_STORAGE_KEY, enabled);
|
||||
set({ sceneEnhancementEnabled: enabled });
|
||||
},
|
||||
}));
|
||||
|
||||
export const useStoryboardPageEnabled = () =>
|
||||
useExperimentalStore((s) => s.enabled && s.storyboardPageEnabled);
|
||||
|
||||
export const useSceneEnhancementEnabled = () =>
|
||||
useExperimentalStore((s) => s.enabled && s.sceneEnhancementEnabled);
|
||||
@@ -18,6 +18,12 @@ export const ExperimentalSettingsPane = ({ onStoryboardPageDisable }: Props) =>
|
||||
const setStoryboardPageEnabled = useExperimentalStore(
|
||||
(s) => s.setStoryboardPageEnabled,
|
||||
);
|
||||
const sceneEnhancementEnabled = useExperimentalStore(
|
||||
(s) => s.sceneEnhancementEnabled,
|
||||
);
|
||||
const setSceneEnhancementEnabled = useExperimentalStore(
|
||||
(s) => s.setSceneEnhancementEnabled,
|
||||
);
|
||||
const [isDisableConfirmOpen, setIsDisableConfirmOpen] = useState(false);
|
||||
|
||||
const fireToggleEvent = (enabled: boolean) => {
|
||||
@@ -27,6 +33,14 @@ export const ExperimentalSettingsPane = ({ onStoryboardPageDisable }: Props) =>
|
||||
});
|
||||
};
|
||||
|
||||
const handleSceneEnhancementToggle = (enabled: boolean) => {
|
||||
setSceneEnhancementEnabled(enabled);
|
||||
gtagEvent("toggle_experimental_feature", {
|
||||
feature: "scene_enhancement",
|
||||
enabled: String(enabled),
|
||||
});
|
||||
};
|
||||
|
||||
const handleStoryboardToggle = (enabled: boolean) => {
|
||||
if (enabled) {
|
||||
setStoryboardPageEnabled(true);
|
||||
@@ -59,6 +73,23 @@ export const ExperimentalSettingsPane = ({ onStoryboardPageDisable }: Props) =>
|
||||
setEnabled={handleStoryboardToggle}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<Label htmlFor="experimental-scene-enhancement">
|
||||
Scene enhancement
|
||||
</Label>
|
||||
<p className="text-xs opacity-70">
|
||||
Adds an Enhance panel to the 3D editor for exporting and
|
||||
re-importing a universal scene descriptor (and glTF/USDZ).
|
||||
In-development; expect rough edges.
|
||||
</p>
|
||||
</div>
|
||||
<Switch
|
||||
enabled={sceneEnhancementEnabled}
|
||||
setEnabled={handleSceneEnhancementToggle}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<ExperimentalConfirmModal
|
||||
isOpen={isDisableConfirmOpen}
|
||||
|
||||
Reference in new issue
Block a user