mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-03 08:57:54 +00:00
feat(webapp): add 3D Object (mesh) and 3D World (splat) create pages
Wire up the new omni_gen mesh + splat endpoints in the frontend and add two new create pages that mirror the existing Create Image / Create Video flow. Backend was already shipped; this is frontend-only. API layer (libs/api, libs/omni-gen): - OmniGenApi: add mesh + splat request/response/model-info types and the six new methods (models/cost/generate for mesh and splat) - Add useOmniGenMeshModels / useOmniGenSplatModels session-cached model hooks Pages (apps/artcraft-webapp): - New /create-object (3D Object) and /create-world (3D World) page trios (page + zustand store + generate/poll api), cloned from create-image - Capability-gated controls per selected model: mesh output type, polygon type, geometry/texture quality, face count, PBR + texture toggles, multi-view (front/back/left/right) and mesh-file inputs; splat panorama + disable-recaption toggles and reference video - Add useMeshCostEstimate / useSplatCostEstimate credit hooks - Register routes in app.tsx and add "3D Object" / "3D World" sidebar entries Shared generation feed (libs/components/generation-list + webapp gallery): - Teach useGenerationJobs the object_generation / splat_generation inference categories, surfaced as the "dimensional" media class - Render 3D result cards from the media file's cover image / screenshot - useGalleryData: optional model-id filter so the object and world feeds only show their own generations - Widen InProgressJob / FailedJob / PendingCard / RecreateSlotContext media class to include "dimensional" Clicking a 3D card opens the existing Viewer3D lightbox (already handles .glb meshes and .spz splats). No backend changes.
This commit is contained in:
1 parent
bec8c7ede4
commit
cc8a124bdb
21 files changed
+3029
-26
No files matched your search
@@ -15,6 +15,8 @@ import { ToastContainer } from "../components/toast/toast";
|
||||
import CreateImage from "../pages/create-image";
|
||||
import CreateVideo from "../pages/create-video";
|
||||
import CreateAudio from "../pages/create-audio";
|
||||
import CreateObject from "../pages/create-object";
|
||||
import CreateWorld from "../pages/create-world";
|
||||
import CreateVFX from "../pages/create-vfx";
|
||||
import PageScene from "../pages/pagescene";
|
||||
import PageDraw from "../pages/pagedraw";
|
||||
@@ -155,6 +157,8 @@ export function App() {
|
||||
<Route path="/create-image" element={<CreateImage />} />
|
||||
<Route path="/create-video" element={<CreateVideo />} />
|
||||
<Route path="/create-audio" element={<CreateAudio />} />
|
||||
<Route path="/create-object" element={<CreateObject />} />
|
||||
<Route path="/create-world" element={<CreateWorld />} />
|
||||
<Route path="/background-change" element={<CreateVFX />} />
|
||||
<Route path="/moodboard" element={<MoodboardPage />} />
|
||||
<Route path="/edit-3d" element={<PageScene />} />
|
||||
|
||||
+5
-1
@@ -65,11 +65,15 @@ function CardRecreateButton({
|
||||
mediaClass,
|
||||
kind,
|
||||
}: RecreateSlotContext) {
|
||||
// Recreate only targets the image/video create pages; 3D generations have no
|
||||
// recreate flow, so map to a valid class for the hook and render nothing.
|
||||
const { isRecreating, handleRecreate } = useRecreateFromPromptToken(
|
||||
promptToken,
|
||||
mediaClass,
|
||||
mediaClass === "dimensional" ? "image" : mediaClass,
|
||||
);
|
||||
|
||||
if (mediaClass === "dimensional") return null;
|
||||
|
||||
if (kind === "failed") {
|
||||
return (
|
||||
<button
|
||||
|
||||
+5
-1
@@ -81,10 +81,14 @@ function RowRecreateButton({
|
||||
mediaClass,
|
||||
kind,
|
||||
}: RecreateSlotContext) {
|
||||
// Recreate only targets the image/video create pages; 3D generations have no
|
||||
// recreate flow, so map to a valid class for the hook and render nothing.
|
||||
const { isRecreating, handleRecreate } = useRecreateFromPromptToken(
|
||||
promptToken,
|
||||
mediaClass,
|
||||
mediaClass === "dimensional" ? "image" : mediaClass,
|
||||
);
|
||||
|
||||
if (mediaClass === "dimensional") return null;
|
||||
return (
|
||||
<Tooltip content="Recreate" position="top">
|
||||
<button
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
faImage,
|
||||
faMusic,
|
||||
faCube,
|
||||
faGlobe,
|
||||
faFilm,
|
||||
faObjectGroup,
|
||||
faWandMagicSparkles,
|
||||
@@ -58,6 +59,8 @@ const CREATE_ITEMS_STATIC: NavItem[] = [
|
||||
{ label: "Image", href: "/create-image", icon: faImage },
|
||||
{ label: "Video", href: "/create-video", icon: faVideo },
|
||||
{ label: "Audio", href: "/create-audio", icon: faMusic },
|
||||
{ label: "3D Object", href: "/create-object", icon: faCube },
|
||||
{ label: "3D World", href: "/create-world", icon: faGlobe },
|
||||
// Edit Image is hidden from the sidebar for now.
|
||||
{ label: "Edit Image", href: "/edit-image", icon: faPencil },
|
||||
{ label: "Edit 3D", href: "/edit-3d", icon: faCube },
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { OmniGenApi } from "@storyteller/api";
|
||||
import type { OmniGenImageRequest, OmniGenVideoRequest } from "@storyteller/api";
|
||||
import type {
|
||||
OmniGenImageRequest,
|
||||
OmniGenVideoRequest,
|
||||
OmniGenMeshRequest,
|
||||
OmniGenSplatRequest,
|
||||
} from "@storyteller/api";
|
||||
import {
|
||||
ModelPage,
|
||||
useSelectedImageModel,
|
||||
@@ -200,3 +205,138 @@ export function useVideoCostEstimate(params: VideoCostParams): number | null {
|
||||
|
||||
return credits;
|
||||
}
|
||||
|
||||
// ── Mesh (3D object) cost estimate hook ──────────────────────────────────
|
||||
|
||||
export interface MeshCostParams {
|
||||
model: string;
|
||||
referenceImageCount: number;
|
||||
hasInputMesh: boolean;
|
||||
meshOutputType?: string | null;
|
||||
polygonType?: string | null;
|
||||
faceCount?: number | null;
|
||||
enablePbr?: boolean | null;
|
||||
enableTexture?: boolean | null;
|
||||
textureQuality?: string | null;
|
||||
geometryQuality?: string | null;
|
||||
}
|
||||
|
||||
export function useMeshCostEstimate(params: MeshCostParams): number | null {
|
||||
const [credits, setCredits] = useState<number | null>(null);
|
||||
const abortRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!params.model) {
|
||||
setCredits(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const id = ++abortRef.current;
|
||||
|
||||
const body: OmniGenMeshRequest = {
|
||||
model: params.model,
|
||||
reference_image_media_tokens:
|
||||
params.referenceImageCount > 0
|
||||
? new Array(params.referenceImageCount).fill("placeholder")
|
||||
: null,
|
||||
input_mesh_media_token: params.hasInputMesh ? "placeholder" : null,
|
||||
mesh_output_type: params.meshOutputType ?? null,
|
||||
polygon_type: params.polygonType ?? null,
|
||||
face_count: params.faceCount ?? null,
|
||||
enable_pbr: params.enablePbr ?? null,
|
||||
enable_texture: params.enableTexture ?? null,
|
||||
texture_quality: params.textureQuality ?? null,
|
||||
geometry_quality: params.geometryQuality ?? null,
|
||||
};
|
||||
|
||||
const api = new OmniGenApi();
|
||||
api.estimateMeshCost(body).then(
|
||||
(response) => {
|
||||
if (id !== abortRef.current) return;
|
||||
if (response.success && response.cost_in_credits != null) {
|
||||
setCredits(response.cost_in_credits);
|
||||
} else {
|
||||
setCredits(null);
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (id !== abortRef.current) return;
|
||||
setCredits(null);
|
||||
},
|
||||
);
|
||||
}, [
|
||||
params.model,
|
||||
params.referenceImageCount,
|
||||
params.hasInputMesh,
|
||||
params.meshOutputType,
|
||||
params.polygonType,
|
||||
params.faceCount,
|
||||
params.enablePbr,
|
||||
params.enableTexture,
|
||||
params.textureQuality,
|
||||
params.geometryQuality,
|
||||
]);
|
||||
|
||||
return credits;
|
||||
}
|
||||
|
||||
// ── Splat (3D world) cost estimate hook ──────────────────────────────────
|
||||
|
||||
export interface SplatCostParams {
|
||||
model: string;
|
||||
referenceImageCount: number;
|
||||
hasReferenceVideo: boolean;
|
||||
isPanoramic?: boolean | null;
|
||||
disableRecaption?: boolean | null;
|
||||
}
|
||||
|
||||
export function useSplatCostEstimate(params: SplatCostParams): number | null {
|
||||
const [credits, setCredits] = useState<number | null>(null);
|
||||
const abortRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!params.model) {
|
||||
setCredits(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const id = ++abortRef.current;
|
||||
|
||||
const body: OmniGenSplatRequest = {
|
||||
model: params.model,
|
||||
reference_image_media_tokens:
|
||||
params.referenceImageCount > 0
|
||||
? new Array(params.referenceImageCount).fill("placeholder")
|
||||
: null,
|
||||
reference_video_media_token: params.hasReferenceVideo
|
||||
? "placeholder"
|
||||
: null,
|
||||
is_panoramic: params.isPanoramic ?? null,
|
||||
disable_recaption: params.disableRecaption ?? null,
|
||||
};
|
||||
|
||||
const api = new OmniGenApi();
|
||||
api.estimateSplatCost(body).then(
|
||||
(response) => {
|
||||
if (id !== abortRef.current) return;
|
||||
if (response.success && response.cost_in_credits != null) {
|
||||
setCredits(response.cost_in_credits);
|
||||
} else {
|
||||
setCredits(null);
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (id !== abortRef.current) return;
|
||||
setCredits(null);
|
||||
},
|
||||
);
|
||||
}, [
|
||||
params.model,
|
||||
params.referenceImageCount,
|
||||
params.hasReferenceVideo,
|
||||
params.isPanoramic,
|
||||
params.disableRecaption,
|
||||
]);
|
||||
|
||||
return credits;
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
import { useRef, useState } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faPlus,
|
||||
faSpinnerThird,
|
||||
faXmark,
|
||||
faCube,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import { MediaUploadApi } from "@storyteller/api";
|
||||
import { FilterEngineCategories } from "@storyteller/api";
|
||||
import { UploaderStates } from "@storyteller/common";
|
||||
import { uploadImage } from "../../../components/prompt-box/upload-image";
|
||||
import type { RefImage } from "../../../components/prompt-box";
|
||||
|
||||
// Multi-view side inputs (front/back/left/right) and the mesh-to-mesh input
|
||||
// file, shown above the prompt box when the selected model supports them. Uses
|
||||
// the same media upload path as the primary reference image row.
|
||||
|
||||
interface MeshInputsRowProps {
|
||||
showMultiView: boolean;
|
||||
showMeshInput: boolean;
|
||||
frontImage?: RefImage;
|
||||
backImage?: RefImage;
|
||||
leftImage?: RefImage;
|
||||
rightImage?: RefImage;
|
||||
inputMesh?: RefImage;
|
||||
onChange: (patch: {
|
||||
frontImage?: RefImage;
|
||||
backImage?: RefImage;
|
||||
leftImage?: RefImage;
|
||||
rightImage?: RefImage;
|
||||
inputMesh?: RefImage;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
const MESH_FILE_ACCEPT = ".glb,.gltf,.fbx,.obj";
|
||||
|
||||
export function MeshInputsRow({
|
||||
showMultiView,
|
||||
showMeshInput,
|
||||
frontImage,
|
||||
backImage,
|
||||
leftImage,
|
||||
rightImage,
|
||||
inputMesh,
|
||||
onChange,
|
||||
}: MeshInputsRowProps) {
|
||||
if (!showMultiView && !showMeshInput) return null;
|
||||
|
||||
return (
|
||||
<div className="glass flex flex-wrap items-end gap-3 rounded-2xl px-3 py-2">
|
||||
{showMultiView && (
|
||||
<>
|
||||
<ImageSlot
|
||||
label="Front"
|
||||
image={frontImage}
|
||||
onChange={(img) => onChange({ frontImage: img })}
|
||||
/>
|
||||
<ImageSlot
|
||||
label="Back"
|
||||
image={backImage}
|
||||
onChange={(img) => onChange({ backImage: img })}
|
||||
/>
|
||||
<ImageSlot
|
||||
label="Left"
|
||||
image={leftImage}
|
||||
onChange={(img) => onChange({ leftImage: img })}
|
||||
/>
|
||||
<ImageSlot
|
||||
label="Right"
|
||||
image={rightImage}
|
||||
onChange={(img) => onChange({ rightImage: img })}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
{showMeshInput && (
|
||||
<MeshFileSlot
|
||||
mesh={inputMesh}
|
||||
onChange={(img) => onChange({ inputMesh: img })}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Slots ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const SLOT_CLASS =
|
||||
"flex aspect-square w-14 items-center justify-center overflow-hidden rounded-lg border-2 border-dashed border-white/25 bg-white/5 transition-all hover:border-white/40 hover:bg-white/10";
|
||||
|
||||
function SlotShell({
|
||||
label,
|
||||
children,
|
||||
}: {
|
||||
label: string;
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-1">
|
||||
{children}
|
||||
<span className="text-[11px] text-white/60">{label}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ImageSlot({
|
||||
label,
|
||||
image,
|
||||
onChange,
|
||||
}: {
|
||||
label: string;
|
||||
image?: RefImage;
|
||||
onChange: (img?: RefImage) => void;
|
||||
}) {
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
|
||||
const handleUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.target.files?.[0];
|
||||
if (!file) return;
|
||||
setUploading(true);
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = async () => {
|
||||
await uploadImage({
|
||||
title: `${label.toLowerCase()}-view-${Math.random().toString(36).substring(2, 10)}`,
|
||||
assetFile: file,
|
||||
progressCallback: (state) => {
|
||||
if (state.status === UploaderStates.success && state.data) {
|
||||
onChange({
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: reader.result as string,
|
||||
file,
|
||||
mediaToken: state.data,
|
||||
});
|
||||
setUploading(false);
|
||||
} else if (
|
||||
state.status === UploaderStates.assetError ||
|
||||
state.status === UploaderStates.imageCreateError
|
||||
) {
|
||||
setUploading(false);
|
||||
}
|
||||
},
|
||||
});
|
||||
if (inputRef.current) inputRef.current.value = "";
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
};
|
||||
|
||||
return (
|
||||
<SlotShell label={label}>
|
||||
<input
|
||||
type="file"
|
||||
ref={inputRef}
|
||||
className="hidden"
|
||||
accept="image/*"
|
||||
onChange={handleUpload}
|
||||
/>
|
||||
{image ? (
|
||||
<div className="group relative aspect-square w-14 overflow-hidden rounded-lg border-2 border-white/30">
|
||||
<img
|
||||
src={image.url}
|
||||
alt={`${label} view`}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
<button
|
||||
onClick={() => onChange(undefined)}
|
||||
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/50 text-white backdrop-blur-md transition-colors hover:bg-black"
|
||||
>
|
||||
<FontAwesomeIcon icon={faXmark} className="h-2.5 w-2.5" />
|
||||
</button>
|
||||
</div>
|
||||
) : uploading ? (
|
||||
<div className={SLOT_CLASS}>
|
||||
<FontAwesomeIcon
|
||||
icon={faSpinnerThird}
|
||||
spin
|
||||
className="h-5 w-5 text-white"
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<button
|
||||
onClick={() => inputRef.current?.click()}
|
||||
className={SLOT_CLASS}
|
||||
>
|
||||
<FontAwesomeIcon icon={faPlus} className="text-xl text-white/80" />
|
||||
</button>
|
||||
)}
|
||||
</SlotShell>
|
||||
);
|
||||
}
|
||||
|
||||
function MeshFileSlot({
|
||||
mesh,
|
||||
onChange,
|
||||
}: {
|
||||
mesh?: RefImage;
|
||||
onChange: (img?: RefImage) => void;
|
||||
}) {
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
|
||||
const handleUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.target.files?.[0];
|
||||
if (!file) return;
|
||||
setUploading(true);
|
||||
try {
|
||||
const api = new MediaUploadApi();
|
||||
const response = await api.UploadNewEngineAsset({
|
||||
file,
|
||||
fileName: file.name || `input-mesh-${Date.now()}`,
|
||||
uuid: crypto.randomUUID(),
|
||||
engine_category: FilterEngineCategories.OBJECT,
|
||||
maybe_title: "input_mesh",
|
||||
});
|
||||
if (response?.success && response.data) {
|
||||
onChange({
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: "",
|
||||
file,
|
||||
mediaToken: response.data,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// ignore — user can retry
|
||||
} finally {
|
||||
setUploading(false);
|
||||
if (inputRef.current) inputRef.current.value = "";
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<SlotShell label="Mesh file">
|
||||
<input
|
||||
type="file"
|
||||
ref={inputRef}
|
||||
className="hidden"
|
||||
accept={MESH_FILE_ACCEPT}
|
||||
onChange={handleUpload}
|
||||
/>
|
||||
{mesh ? (
|
||||
<div className="group relative flex aspect-square w-14 items-center justify-center overflow-hidden rounded-lg border-2 border-white/30 bg-white/10">
|
||||
<FontAwesomeIcon icon={faCube} className="h-5 w-5 text-white/80" />
|
||||
<button
|
||||
onClick={() => onChange(undefined)}
|
||||
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/50 text-white backdrop-blur-md transition-colors hover:bg-black"
|
||||
>
|
||||
<FontAwesomeIcon icon={faXmark} className="h-2.5 w-2.5" />
|
||||
</button>
|
||||
</div>
|
||||
) : uploading ? (
|
||||
<div className={SLOT_CLASS}>
|
||||
<FontAwesomeIcon
|
||||
icon={faSpinnerThird}
|
||||
spin
|
||||
className="h-5 w-5 text-white"
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<button
|
||||
onClick={() => inputRef.current?.click()}
|
||||
className={SLOT_CLASS}
|
||||
>
|
||||
<FontAwesomeIcon icon={faCube} className="text-xl text-white/80" />
|
||||
</button>
|
||||
)}
|
||||
</SlotShell>
|
||||
);
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import {
|
||||
faCube,
|
||||
faObjectGroup,
|
||||
faWandMagicSparkles,
|
||||
faImage,
|
||||
faGem,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import type { IconDefinition } from "@fortawesome/fontawesome-svg-core";
|
||||
import { PopoverItem, PopoverMenu } from "@storyteller/ui-popover";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
|
||||
// ── Enum label maps (backend value → human label) ──────────────────────────
|
||||
|
||||
const MESH_OUTPUT_TYPE_LABELS: Record<string, string> = {
|
||||
normal: "Standard",
|
||||
low_poly: "Low Poly",
|
||||
geometry: "Geometry only",
|
||||
};
|
||||
|
||||
const POLYGON_TYPE_LABELS: Record<string, string> = {
|
||||
triangle: "Triangle",
|
||||
quad: "Quad",
|
||||
};
|
||||
|
||||
const MESH_QUALITY_LABELS: Record<string, string> = {
|
||||
standard: "Standard",
|
||||
detailed: "Detailed",
|
||||
};
|
||||
|
||||
// Preset target face counts (poly budget). Backend accepts any u64; these are
|
||||
// sensible presets exposed in the UI.
|
||||
const FACE_COUNT_PRESETS: number[] = [10000, 30000, 50000, 100000, 200000];
|
||||
|
||||
// ── Generic enum picker ────────────────────────────────────────────────────
|
||||
|
||||
interface EnumPickerProps {
|
||||
options: string[];
|
||||
current?: string;
|
||||
onSelect: (value: string) => void;
|
||||
labels: Record<string, string>;
|
||||
tooltip: string;
|
||||
panelTitle: string;
|
||||
icon: IconDefinition;
|
||||
}
|
||||
|
||||
function EnumPicker({
|
||||
options,
|
||||
current,
|
||||
onSelect,
|
||||
labels,
|
||||
tooltip,
|
||||
panelTitle,
|
||||
icon,
|
||||
}: EnumPickerProps) {
|
||||
const labelToValue: Record<string, string> = Object.fromEntries(
|
||||
options.map((v) => [labels[v] ?? v, v]),
|
||||
);
|
||||
|
||||
const items: PopoverItem[] = options.map((v) => ({
|
||||
label: labels[v] ?? v,
|
||||
selected: current === v,
|
||||
}));
|
||||
|
||||
return (
|
||||
<Tooltip content={tooltip} position="top" className="z-50" closeOnClick>
|
||||
<PopoverMenu
|
||||
items={items}
|
||||
onSelect={(item) => {
|
||||
const value = labelToValue[item.label];
|
||||
if (value) onSelect(value);
|
||||
}}
|
||||
mode="toggle"
|
||||
panelTitle={panelTitle}
|
||||
triggerIcon={<FontAwesomeIcon icon={icon} className="h-3.5 w-3.5" />}
|
||||
/>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Concrete pickers ───────────────────────────────────────────────────────
|
||||
|
||||
export const MeshOutputTypePicker = (props: {
|
||||
options: string[];
|
||||
current?: string;
|
||||
onSelect: (value: string) => void;
|
||||
}) => (
|
||||
<EnumPicker
|
||||
{...props}
|
||||
labels={MESH_OUTPUT_TYPE_LABELS}
|
||||
tooltip="Output type"
|
||||
panelTitle="Output type"
|
||||
icon={faCube}
|
||||
/>
|
||||
);
|
||||
|
||||
export const PolygonTypePicker = (props: {
|
||||
options: string[];
|
||||
current?: string;
|
||||
onSelect: (value: string) => void;
|
||||
}) => (
|
||||
<EnumPicker
|
||||
{...props}
|
||||
labels={POLYGON_TYPE_LABELS}
|
||||
tooltip="Polygon type"
|
||||
panelTitle="Polygon type"
|
||||
icon={faObjectGroup}
|
||||
/>
|
||||
);
|
||||
|
||||
export const GeometryQualityPicker = (props: {
|
||||
options: string[];
|
||||
current?: string;
|
||||
onSelect: (value: string) => void;
|
||||
}) => (
|
||||
<EnumPicker
|
||||
{...props}
|
||||
labels={MESH_QUALITY_LABELS}
|
||||
tooltip="Geometry quality"
|
||||
panelTitle="Geometry quality"
|
||||
icon={faWandMagicSparkles}
|
||||
/>
|
||||
);
|
||||
|
||||
export const TextureQualityPicker = (props: {
|
||||
options: string[];
|
||||
current?: string;
|
||||
onSelect: (value: string) => void;
|
||||
}) => (
|
||||
<EnumPicker
|
||||
{...props}
|
||||
labels={MESH_QUALITY_LABELS}
|
||||
tooltip="Texture quality"
|
||||
panelTitle="Texture quality"
|
||||
icon={faImage}
|
||||
/>
|
||||
);
|
||||
|
||||
// Standard two-value quality option lists ("standard" / "detailed").
|
||||
export const QUALITY_OPTIONS = ["standard", "detailed"];
|
||||
|
||||
export const FaceCountPicker = ({
|
||||
current,
|
||||
onSelect,
|
||||
}: {
|
||||
current?: number;
|
||||
onSelect: (value: number | undefined) => void;
|
||||
}) => {
|
||||
const items: PopoverItem[] = [
|
||||
{ label: "Auto", selected: current == null },
|
||||
...FACE_COUNT_PRESETS.map((n) => ({
|
||||
label: `${n / 1000}k`,
|
||||
selected: current === n,
|
||||
})),
|
||||
];
|
||||
|
||||
return (
|
||||
<Tooltip content="Face count" position="top" className="z-50" closeOnClick>
|
||||
<PopoverMenu
|
||||
items={items}
|
||||
onSelect={(item) => {
|
||||
if (item.label === "Auto") {
|
||||
onSelect(undefined);
|
||||
return;
|
||||
}
|
||||
const n = parseFloat(item.label) * 1000;
|
||||
if (!Number.isNaN(n)) onSelect(n);
|
||||
}}
|
||||
mode="toggle"
|
||||
panelTitle="Face count"
|
||||
triggerIcon={<FontAwesomeIcon icon={faGem} className="h-3.5 w-3.5" />}
|
||||
/>
|
||||
</Tooltip>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,169 @@
|
||||
import { create } from "zustand";
|
||||
import { persist } from "zustand/middleware";
|
||||
import type { RecreatePayload } from "../../lib/recreate";
|
||||
import type { RefImage } from "../../components/prompt-box";
|
||||
|
||||
// A generated 3D asset (mesh). Shape mirrors the image store's GeneratedImage so
|
||||
// the shared polling helpers can be reused; `cdn_url` here is the .glb file.
|
||||
export interface GeneratedAsset {
|
||||
media_token: string;
|
||||
cdn_url: string;
|
||||
maybe_thumbnail_template?: string;
|
||||
}
|
||||
|
||||
export type MeshBatch = {
|
||||
id: string;
|
||||
prompt: string;
|
||||
status: "pending" | "complete" | "failed";
|
||||
assets: GeneratedAsset[];
|
||||
createdAt: number;
|
||||
modelLabel: string;
|
||||
jobToken?: string;
|
||||
failureReason?: string;
|
||||
};
|
||||
|
||||
export type MeshUiState = {
|
||||
selectedModelId: string | null;
|
||||
prompt: string;
|
||||
meshOutputType: string | undefined;
|
||||
polygonType: string | undefined;
|
||||
faceCount: number | undefined;
|
||||
enablePbr: boolean | undefined;
|
||||
enableTexture: boolean | undefined;
|
||||
textureQuality: string | undefined;
|
||||
geometryQuality: string | undefined;
|
||||
};
|
||||
|
||||
// Multi-view side inputs + a mesh-to-mesh input file. Not persisted (File
|
||||
// handles + data URLs aren't serializable / can exceed the storage quota).
|
||||
export type MeshInputsState = {
|
||||
frontImage: RefImage | undefined;
|
||||
backImage: RefImage | undefined;
|
||||
leftImage: RefImage | undefined;
|
||||
rightImage: RefImage | undefined;
|
||||
inputMesh: RefImage | undefined;
|
||||
};
|
||||
|
||||
type CreateObjectState = {
|
||||
batches: MeshBatch[];
|
||||
ui: MeshUiState;
|
||||
referenceImages: RefImage[];
|
||||
inputs: MeshInputsState;
|
||||
pendingRecreate: RecreatePayload | null;
|
||||
setUi: (patch: Partial<MeshUiState>) => void;
|
||||
setReferenceImages: (images: RefImage[]) => void;
|
||||
setInputs: (patch: Partial<MeshInputsState>) => void;
|
||||
setPendingRecreate: (payload: RecreatePayload | null) => void;
|
||||
consumePendingRecreate: () => RecreatePayload | null;
|
||||
startBatch: (prompt: string, modelLabel: string) => string;
|
||||
setBatchJobToken: (batchId: string, jobToken: string) => void;
|
||||
completeBatch: (batchId: string, assets: GeneratedAsset[]) => void;
|
||||
failBatch: (batchId: string, reason?: string) => void;
|
||||
dismissBatch: (id: string) => void;
|
||||
reset: () => void;
|
||||
};
|
||||
|
||||
const DEFAULT_UI: MeshUiState = {
|
||||
selectedModelId: null,
|
||||
prompt: "",
|
||||
meshOutputType: undefined,
|
||||
polygonType: undefined,
|
||||
faceCount: undefined,
|
||||
enablePbr: undefined,
|
||||
enableTexture: undefined,
|
||||
textureQuality: undefined,
|
||||
geometryQuality: undefined,
|
||||
};
|
||||
|
||||
const DEFAULT_INPUTS: MeshInputsState = {
|
||||
frontImage: undefined,
|
||||
backImage: undefined,
|
||||
leftImage: undefined,
|
||||
rightImage: undefined,
|
||||
inputMesh: undefined,
|
||||
};
|
||||
|
||||
export const useCreateObjectStore = create<CreateObjectState>()(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
batches: [],
|
||||
ui: { ...DEFAULT_UI },
|
||||
referenceImages: [],
|
||||
inputs: { ...DEFAULT_INPUTS },
|
||||
pendingRecreate: null,
|
||||
|
||||
setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })),
|
||||
|
||||
setReferenceImages: (images) => set({ referenceImages: images }),
|
||||
|
||||
setInputs: (patch) => set((s) => ({ inputs: { ...s.inputs, ...patch } })),
|
||||
|
||||
setPendingRecreate: (payload) => set({ pendingRecreate: payload }),
|
||||
|
||||
consumePendingRecreate: () => {
|
||||
const payload = get().pendingRecreate;
|
||||
if (payload) set({ pendingRecreate: null });
|
||||
return payload;
|
||||
},
|
||||
|
||||
startBatch: (prompt, modelLabel) => {
|
||||
const id = crypto.randomUUID();
|
||||
const batch: MeshBatch = {
|
||||
id,
|
||||
prompt,
|
||||
status: "pending",
|
||||
assets: [],
|
||||
createdAt: Date.now(),
|
||||
modelLabel,
|
||||
};
|
||||
set((s) => ({ batches: [...s.batches, batch] }));
|
||||
return id;
|
||||
},
|
||||
|
||||
setBatchJobToken: (batchId, jobToken) => {
|
||||
set((s) => ({
|
||||
batches: s.batches.map((b) =>
|
||||
b.id === batchId ? { ...b, jobToken } : b,
|
||||
),
|
||||
}));
|
||||
},
|
||||
|
||||
completeBatch: (batchId, assets) => {
|
||||
set((s) => ({
|
||||
batches: s.batches.map((b) =>
|
||||
b.id === batchId
|
||||
? { ...b, status: "complete" as const, assets }
|
||||
: b,
|
||||
),
|
||||
}));
|
||||
},
|
||||
|
||||
failBatch: (batchId, reason) => {
|
||||
set((s) => ({
|
||||
batches: s.batches.map((b) =>
|
||||
b.id === batchId
|
||||
? { ...b, status: "failed" as const, failureReason: reason }
|
||||
: b,
|
||||
),
|
||||
}));
|
||||
},
|
||||
|
||||
dismissBatch: (id) => {
|
||||
set((s) => ({ batches: s.batches.filter((b) => b.id !== id) }));
|
||||
},
|
||||
|
||||
reset: () => set({ batches: [] }),
|
||||
}),
|
||||
{
|
||||
name: "artcraft-object-prompt",
|
||||
// Persist only the lightweight settings; reference images / input files
|
||||
// are excluded (not serializable, can exceed the localStorage quota).
|
||||
partialize: (state) => ({ ui: state.ui }),
|
||||
merge: (persisted, current) => {
|
||||
const persistedUi = (persisted as { ui?: Partial<MeshUiState> } | null)
|
||||
?.ui;
|
||||
return { ...current, ui: { ...current.ui, ...(persistedUi ?? {}) } };
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
@@ -0,0 +1,646 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { FilterMediaClasses } from "@storyteller/api";
|
||||
import type { OmniGenMeshModelInfo } from "@storyteller/api";
|
||||
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
import { Button, ToggleButton } from "@storyteller/ui-button";
|
||||
import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal";
|
||||
import {
|
||||
faSparkles,
|
||||
faGem,
|
||||
faImage,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
PromptBox,
|
||||
ImagePromptRow,
|
||||
MobilePromptForm,
|
||||
MobileSelectField,
|
||||
type RefImage,
|
||||
} from "../../components/prompt-box";
|
||||
import {
|
||||
GenerationGallery,
|
||||
useGalleryData,
|
||||
useGenerationJobs,
|
||||
useAuthCheck,
|
||||
usePromptHeight,
|
||||
useLightboxNav,
|
||||
CreateMediaPageShell,
|
||||
} from "../../components/generation-gallery";
|
||||
import { Lightbox } from "../../components/lightbox/lightbox";
|
||||
import { useCreateObjectStore } from "./create-object-store";
|
||||
import { enqueueMeshGeneration, startPolling } from "./generate-object-api";
|
||||
import {
|
||||
MeshOutputTypePicker,
|
||||
PolygonTypePicker,
|
||||
GeometryQualityPicker,
|
||||
TextureQualityPicker,
|
||||
FaceCountPicker,
|
||||
QUALITY_OPTIONS,
|
||||
} from "./components/MeshOptionPickers";
|
||||
import { MeshInputsRow } from "./components/MeshInputsRow";
|
||||
import { useMeshCostEstimate } from "../../lib/cost-estimate-api";
|
||||
import { resolveModelOption } from "../../lib/resolve-model-setting";
|
||||
import {
|
||||
useOmniGenMeshModels,
|
||||
OMNI_GENERATE_OUTAGE_MESSAGE,
|
||||
} from "@storyteller/omni-gen";
|
||||
import {
|
||||
getCreatorIconPathForModelId,
|
||||
getModelDescription,
|
||||
getModelInfo,
|
||||
} from "@storyteller/model-list";
|
||||
import { toast } from "../../components/toast/toast";
|
||||
import { useSignupCta } from "../../components/signup-cta-modal";
|
||||
import { useInsufficientCredits } from "../../components/insufficient-credits-modal";
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_MODEL_ID = "hunyuan_3d_3";
|
||||
const DIMENSIONAL_FILTER = [FilterMediaClasses.DIMENSIONAL];
|
||||
|
||||
let _modelLookup = new Map<string, OmniGenMeshModelInfo>();
|
||||
|
||||
function buildModelPopoverItems(
|
||||
models: OmniGenMeshModelInfo[],
|
||||
selectedId: string,
|
||||
): PopoverItem[] {
|
||||
_modelLookup = new Map(models.map((m) => [m.model, m]));
|
||||
return models.map((model) => ({
|
||||
label: model.full_name || model.model,
|
||||
selected: model.model === selectedId,
|
||||
description: getModelDescription(model.model, model.extra_info_short),
|
||||
info: getModelInfo(model.model, model.extra_info) || undefined,
|
||||
icon: (
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(model.model)}
|
||||
alt={`${model.model} logo`}
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
),
|
||||
action: model.model,
|
||||
}));
|
||||
}
|
||||
|
||||
// ── Component ────────────────────────────────────────────────────────────
|
||||
|
||||
export default function CreateObject() {
|
||||
const { user, authChecked } = useAuthCheck();
|
||||
const { loggedIn, openSignupCta } = useSignupCta();
|
||||
const openInsufficientCredits = useInsufficientCredits();
|
||||
const { promptBoxRef, promptHeight } = usePromptHeight();
|
||||
|
||||
const { models: apiModels } = useOmniGenMeshModels();
|
||||
|
||||
const ui = useCreateObjectStore((s) => s.ui);
|
||||
const setUi = useCreateObjectStore((s) => s.setUi);
|
||||
|
||||
const selectedModel = useMemo((): OmniGenMeshModelInfo | undefined => {
|
||||
if (!apiModels.length) return undefined;
|
||||
if (ui.selectedModelId) {
|
||||
return (
|
||||
apiModels.find((m) => m.model === ui.selectedModelId) ??
|
||||
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ??
|
||||
apiModels[0]
|
||||
);
|
||||
}
|
||||
return apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0];
|
||||
}, [apiModels, ui.selectedModelId]);
|
||||
|
||||
const prompt = ui.prompt;
|
||||
const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]);
|
||||
|
||||
// Capability flags for the selected model.
|
||||
const supportsText = !!selectedModel?.text_prompt_supported;
|
||||
const supportsImage =
|
||||
!!selectedModel?.image_input_supported ||
|
||||
!!selectedModel?.sketch_input_supported;
|
||||
const supportsMultiView = !!selectedModel?.multi_view_supported;
|
||||
const supportsMeshInput = !!selectedModel?.mesh_input_supported;
|
||||
const outputTypes = selectedModel?.mesh_output_types ?? [];
|
||||
const polygonTypes = selectedModel?.polygon_types ?? [];
|
||||
const supportsGeometryQuality = !!selectedModel?.geometry_quality_supported;
|
||||
const supportsTextureQuality = !!selectedModel?.texture_quality_supported;
|
||||
const supportsFaceCount = !!selectedModel?.face_count_supported;
|
||||
const supportsPbr = !!selectedModel?.pbr_supported;
|
||||
const supportsTextureToggle = !!selectedModel?.texture_toggle_supported;
|
||||
|
||||
// Settings are sticky across model switches, resolved against the current
|
||||
// model's supported options at read time (see lib/resolve-model-setting).
|
||||
const meshOutputType = resolveModelOption(ui.meshOutputType, outputTypes, outputTypes[0]);
|
||||
const polygonType = resolveModelOption(ui.polygonType, polygonTypes, polygonTypes[0]);
|
||||
const geometryQuality = resolveModelOption(
|
||||
ui.geometryQuality,
|
||||
supportsGeometryQuality ? QUALITY_OPTIONS : [],
|
||||
"standard",
|
||||
);
|
||||
const textureQuality = resolveModelOption(
|
||||
ui.textureQuality,
|
||||
supportsTextureQuality ? QUALITY_OPTIONS : [],
|
||||
"standard",
|
||||
);
|
||||
// Toggles default on when supported unless the user turned them off.
|
||||
const enablePbr = supportsPbr ? (ui.enablePbr ?? false) : false;
|
||||
const enableTexture = supportsTextureToggle ? (ui.enableTexture ?? true) : true;
|
||||
|
||||
const [isGenerating, setIsGenerating] = useState(false);
|
||||
const referenceImages = useCreateObjectStore((s) => s.referenceImages);
|
||||
const setReferenceImages = useCreateObjectStore((s) => s.setReferenceImages);
|
||||
const inputs = useCreateObjectStore((s) => s.inputs);
|
||||
const setInputs = useCreateObjectStore((s) => s.setInputs);
|
||||
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
|
||||
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isImagePickerOpen) setPickerSelectedIds([]);
|
||||
}, [isImagePickerOpen]);
|
||||
|
||||
// Batch store
|
||||
const startBatch = useCreateObjectStore((s) => s.startBatch);
|
||||
const setBatchJobToken = useCreateObjectStore((s) => s.setBatchJobToken);
|
||||
const completeBatch = useCreateObjectStore((s) => s.completeBatch);
|
||||
const failBatch = useCreateObjectStore((s) => s.failBatch);
|
||||
const dismissBatch = useCreateObjectStore((s) => s.dismissBatch);
|
||||
const pollingCleanupsRef = useRef<Map<string, () => void>>(new Map());
|
||||
|
||||
// Jobs + gallery (only this page's own — mesh objects, not splats)
|
||||
const meshModelIds = useMemo(
|
||||
() => apiModels.map((m) => m.model),
|
||||
[apiModels],
|
||||
);
|
||||
const jobs = useGenerationJobs({ mediaType: "object", enabled: !!user });
|
||||
const gallery = useGalleryData({
|
||||
username: user?.username ?? null,
|
||||
filterMediaClasses: DIMENSIONAL_FILTER,
|
||||
filterModelIds: meshModelIds,
|
||||
excludeUploads: true,
|
||||
});
|
||||
|
||||
const newlyCompletedTokens = useMemo(
|
||||
() => new Set(jobs.newlyCompleted.map((i) => i.id)),
|
||||
[jobs.newlyCompleted],
|
||||
);
|
||||
|
||||
const flatItems = useMemo(() => {
|
||||
const filtered = gallery.items.filter(
|
||||
(i) => !newlyCompletedTokens.has(i.id),
|
||||
);
|
||||
return [...jobs.newlyCompleted, ...filtered];
|
||||
}, [jobs.newlyCompleted, gallery.items, newlyCompletedTokens]);
|
||||
|
||||
const lightbox = useLightboxNav(flatItems);
|
||||
|
||||
const estimatedCredits = useMeshCostEstimate({
|
||||
model: selectedModel?.model ?? "",
|
||||
referenceImageCount: referenceImages.length,
|
||||
hasInputMesh: !!inputs.inputMesh,
|
||||
meshOutputType: outputTypes.length ? meshOutputType : undefined,
|
||||
polygonType: polygonTypes.length ? polygonType : undefined,
|
||||
faceCount: supportsFaceCount ? ui.faceCount : undefined,
|
||||
enablePbr: supportsPbr ? enablePbr : undefined,
|
||||
enableTexture: supportsTextureToggle ? enableTexture : undefined,
|
||||
textureQuality: supportsTextureQuality ? textureQuality : undefined,
|
||||
geometryQuality: supportsGeometryQuality ? geometryQuality : undefined,
|
||||
});
|
||||
|
||||
const modelItems = useMemo(
|
||||
() => buildModelPopoverItems(apiModels, selectedModel?.model ?? ""),
|
||||
[apiModels, selectedModel?.model],
|
||||
);
|
||||
|
||||
const hasContent =
|
||||
jobs.inProgress.length > 0 ||
|
||||
jobs.failed.length > 0 ||
|
||||
jobs.newlyCompleted.length > 0 ||
|
||||
gallery.items.length > 0 ||
|
||||
gallery.isInitialLoading;
|
||||
|
||||
const hasAnyImage =
|
||||
referenceImages.length > 0 ||
|
||||
!!inputs.frontImage ||
|
||||
!!inputs.backImage ||
|
||||
!!inputs.leftImage ||
|
||||
!!inputs.rightImage;
|
||||
|
||||
const canGenerate =
|
||||
!!selectedModel &&
|
||||
!isGenerating &&
|
||||
(supportsMeshInput
|
||||
? !!inputs.inputMesh
|
||||
: (supportsText && prompt.trim().length > 0) || hasAnyImage);
|
||||
|
||||
// ── Effects ──────────────────────────────────────────────────────────────
|
||||
|
||||
// Resume polling for pending batches (page revisit).
|
||||
useEffect(() => {
|
||||
const cleanups = pollingCleanupsRef.current;
|
||||
const pendingBatches = useCreateObjectStore
|
||||
.getState()
|
||||
.batches.filter((b) => b.status === "pending" && b.jobToken);
|
||||
|
||||
for (const batch of pendingBatches) {
|
||||
if (cleanups.has(batch.id)) continue;
|
||||
const stop = startPolling(
|
||||
batch.jobToken!,
|
||||
(assets) => {
|
||||
completeBatch(batch.id, assets);
|
||||
cleanups.delete(batch.id);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
(reason) => {
|
||||
failBatch(batch.id, reason);
|
||||
cleanups.delete(batch.id);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
);
|
||||
cleanups.set(batch.id, stop);
|
||||
}
|
||||
|
||||
return () => {
|
||||
cleanups.forEach((stop) => stop());
|
||||
cleanups.clear();
|
||||
};
|
||||
}, [completeBatch, failBatch]);
|
||||
|
||||
// ── Handlers ─────────────────────────────────────────────────────────────
|
||||
|
||||
const handleModelChange = useCallback(
|
||||
(item: PopoverItem) => {
|
||||
const model = item.action ? _modelLookup.get(item.action) : undefined;
|
||||
if (!model) return;
|
||||
setUi({ selectedModelId: model.model });
|
||||
},
|
||||
[setUi],
|
||||
);
|
||||
|
||||
const handleLibraryImageSelect = useCallback(
|
||||
(items: GalleryItem[]) => {
|
||||
const newImages: RefImage[] = items.slice(0, 1).map((item) => ({
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: item.thumbnail || item.fullImage || "",
|
||||
fullUrl: item.fullImage || undefined,
|
||||
file: new File([], "library-image"),
|
||||
mediaToken: item.id,
|
||||
}));
|
||||
setReferenceImages(newImages);
|
||||
setIsImagePickerOpen(false);
|
||||
},
|
||||
[setReferenceImages],
|
||||
);
|
||||
|
||||
const handleGenerate = useCallback(async () => {
|
||||
if (!loggedIn) {
|
||||
openSignupCta();
|
||||
return;
|
||||
}
|
||||
if (!canGenerate || !selectedModel) return;
|
||||
|
||||
setIsGenerating(true);
|
||||
const batchId = startBatch(
|
||||
prompt || selectedModel.full_name || selectedModel.model,
|
||||
selectedModel.full_name ?? selectedModel.model,
|
||||
);
|
||||
|
||||
try {
|
||||
const referenceImageMediaTokens = supportsImage
|
||||
? referenceImages
|
||||
.map((img) => img.mediaToken)
|
||||
.filter((t): t is string => typeof t === "string" && t.length > 0)
|
||||
: undefined;
|
||||
|
||||
const result = await enqueueMeshGeneration({
|
||||
model: selectedModel.model,
|
||||
prompt: supportsText && prompt.trim() ? prompt.trim() : undefined,
|
||||
referenceImageMediaTokens: referenceImageMediaTokens?.length
|
||||
? referenceImageMediaTokens
|
||||
: undefined,
|
||||
frontImageMediaToken: supportsMultiView
|
||||
? inputs.frontImage?.mediaToken
|
||||
: undefined,
|
||||
backImageMediaToken: supportsMultiView
|
||||
? inputs.backImage?.mediaToken
|
||||
: undefined,
|
||||
leftImageMediaToken: supportsMultiView
|
||||
? inputs.leftImage?.mediaToken
|
||||
: undefined,
|
||||
rightImageMediaToken: supportsMultiView
|
||||
? inputs.rightImage?.mediaToken
|
||||
: undefined,
|
||||
inputMeshMediaToken: supportsMeshInput
|
||||
? inputs.inputMesh?.mediaToken
|
||||
: undefined,
|
||||
meshOutputType: outputTypes.length ? meshOutputType : undefined,
|
||||
polygonType: polygonTypes.length ? polygonType : undefined,
|
||||
faceCount: supportsFaceCount ? ui.faceCount : undefined,
|
||||
enablePbr: supportsPbr ? enablePbr : undefined,
|
||||
enableTexture: supportsTextureToggle ? enableTexture : undefined,
|
||||
textureQuality: supportsTextureQuality ? textureQuality : undefined,
|
||||
geometryQuality: supportsGeometryQuality ? geometryQuality : undefined,
|
||||
});
|
||||
|
||||
if (!result.success || !result.jobToken) {
|
||||
if (result.errorCode === 402) {
|
||||
dismissBatch(batchId);
|
||||
openInsufficientCredits();
|
||||
} else {
|
||||
if (result.errorCode != null && result.errorCode >= 500) {
|
||||
toast.error(OMNI_GENERATE_OUTAGE_MESSAGE);
|
||||
}
|
||||
failBatch(batchId, result.error ?? "Failed to start generation");
|
||||
}
|
||||
setIsGenerating(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setBatchJobToken(batchId, result.jobToken);
|
||||
window.dispatchEvent(new Event("credits-change"));
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
|
||||
const stopPolling = startPolling(
|
||||
result.jobToken,
|
||||
(assets) => {
|
||||
completeBatch(batchId, assets);
|
||||
pollingCleanupsRef.current.delete(batchId);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
(reason) => {
|
||||
failBatch(batchId, reason);
|
||||
pollingCleanupsRef.current.delete(batchId);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
);
|
||||
pollingCleanupsRef.current.set(batchId, stopPolling);
|
||||
} catch {
|
||||
failBatch(batchId, "Network error - please try again");
|
||||
} finally {
|
||||
setIsGenerating(false);
|
||||
}
|
||||
}, [
|
||||
loggedIn,
|
||||
openSignupCta,
|
||||
openInsufficientCredits,
|
||||
canGenerate,
|
||||
selectedModel,
|
||||
prompt,
|
||||
supportsText,
|
||||
supportsImage,
|
||||
supportsMultiView,
|
||||
supportsMeshInput,
|
||||
referenceImages,
|
||||
inputs,
|
||||
outputTypes,
|
||||
meshOutputType,
|
||||
polygonTypes,
|
||||
polygonType,
|
||||
supportsFaceCount,
|
||||
ui.faceCount,
|
||||
supportsPbr,
|
||||
enablePbr,
|
||||
supportsTextureToggle,
|
||||
enableTexture,
|
||||
supportsTextureQuality,
|
||||
textureQuality,
|
||||
supportsGeometryQuality,
|
||||
geometryQuality,
|
||||
startBatch,
|
||||
setBatchJobToken,
|
||||
completeBatch,
|
||||
failBatch,
|
||||
dismissBatch,
|
||||
]);
|
||||
|
||||
const placeholder = supportsText
|
||||
? "Describe the 3D object you want..."
|
||||
: supportsMeshInput
|
||||
? "Upload a mesh file below to process"
|
||||
: "Add a reference image below";
|
||||
|
||||
// Only render the extra-inputs row when the model actually supports one of its
|
||||
// slots, otherwise it renders empty but still flags a "row above" the prompt
|
||||
// box (flat-top visual glitch).
|
||||
const meshInputsRow =
|
||||
supportsMultiView || supportsMeshInput ? (
|
||||
<MeshInputsRow
|
||||
showMultiView={supportsMultiView}
|
||||
showMeshInput={supportsMeshInput}
|
||||
frontImage={inputs.frontImage}
|
||||
backImage={inputs.backImage}
|
||||
leftImage={inputs.leftImage}
|
||||
rightImage={inputs.rightImage}
|
||||
inputMesh={inputs.inputMesh}
|
||||
onChange={setInputs}
|
||||
/>
|
||||
) : undefined;
|
||||
|
||||
// ── Option controls (shared desktop + mobile) ──────────────────────────
|
||||
|
||||
const optionControls = (
|
||||
<>
|
||||
{outputTypes.length > 0 && (
|
||||
<MeshOutputTypePicker
|
||||
options={outputTypes}
|
||||
current={meshOutputType}
|
||||
onSelect={(v) => setUi({ meshOutputType: v })}
|
||||
/>
|
||||
)}
|
||||
{polygonTypes.length > 0 && (
|
||||
<PolygonTypePicker
|
||||
options={polygonTypes}
|
||||
current={polygonType}
|
||||
onSelect={(v) => setUi({ polygonType: v })}
|
||||
/>
|
||||
)}
|
||||
{supportsGeometryQuality && (
|
||||
<GeometryQualityPicker
|
||||
options={QUALITY_OPTIONS}
|
||||
current={geometryQuality}
|
||||
onSelect={(v) => setUi({ geometryQuality: v })}
|
||||
/>
|
||||
)}
|
||||
{supportsTextureQuality && (
|
||||
<TextureQualityPicker
|
||||
options={QUALITY_OPTIONS}
|
||||
current={textureQuality}
|
||||
onSelect={(v) => setUi({ textureQuality: v })}
|
||||
/>
|
||||
)}
|
||||
{supportsFaceCount && (
|
||||
<FaceCountPicker
|
||||
current={ui.faceCount}
|
||||
onSelect={(v) => setUi({ faceCount: v })}
|
||||
/>
|
||||
)}
|
||||
{supportsPbr && (
|
||||
<Tooltip content="PBR materials" position="top" closeOnClick>
|
||||
<ToggleButton
|
||||
isActive={enablePbr}
|
||||
icon={faGem}
|
||||
activeIcon={faGem}
|
||||
label={enablePbr ? "PBR" : "PBR off"}
|
||||
onClick={() => setUi({ enablePbr: !enablePbr })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
{supportsTextureToggle && (
|
||||
<Tooltip content="Generate textures" position="top" closeOnClick>
|
||||
<ToggleButton
|
||||
isActive={enableTexture}
|
||||
icon={faImage}
|
||||
activeIcon={faImage}
|
||||
label={enableTexture ? "Texture" : "No texture"}
|
||||
onClick={() => setUi({ enableTexture: !enableTexture })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
// ── Mobile form ─────────────────────────────────────────────────────────
|
||||
|
||||
const mobileForm = (
|
||||
<MobilePromptForm
|
||||
prompt={prompt}
|
||||
onPromptChange={setPrompt}
|
||||
onSubmit={handleGenerate}
|
||||
isSubmitting={isGenerating}
|
||||
credits={estimatedCredits}
|
||||
placeholder={placeholder}
|
||||
autoAdvance={loggedIn && canGenerate}
|
||||
modelField={
|
||||
<MobileSelectField
|
||||
label="Model"
|
||||
title="Select Model"
|
||||
items={modelItems}
|
||||
onSelect={handleModelChange}
|
||||
/>
|
||||
}
|
||||
frames={
|
||||
supportsImage ? (
|
||||
<ImagePromptRow
|
||||
maxImagePromptCount={1}
|
||||
referenceImages={referenceImages}
|
||||
setReferenceImages={setReferenceImages}
|
||||
onPickFromLibrary={() => setIsImagePickerOpen(true)}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
settingsFields={meshInputsRow}
|
||||
/>
|
||||
);
|
||||
|
||||
// ── Render ────────────────────────────────────────────────────────────
|
||||
|
||||
return (
|
||||
<CreateMediaPageShell
|
||||
title="3D Object - ArtCraft"
|
||||
description="Generate 3D objects with ArtCraft"
|
||||
authChecked={authChecked}
|
||||
hasContent={hasContent}
|
||||
emptyStateTitle="Create 3D Object"
|
||||
emptyStateSubtitle="Turn a prompt or image into a 3D model."
|
||||
emptyStateCta={
|
||||
loggedIn ? undefined : (
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={openSignupCta}
|
||||
icon={faSparkles}
|
||||
className="h-12 px-6 text-base font-semibold rounded-full"
|
||||
>
|
||||
Sign up to create
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
bottomOffset={promptHeight + 24}
|
||||
modelItems={modelItems}
|
||||
onModelChange={handleModelChange}
|
||||
promptForm={mobileForm}
|
||||
gridContent={
|
||||
<GenerationGallery
|
||||
inProgressJobs={jobs.inProgress}
|
||||
failedJobs={jobs.failed}
|
||||
onDismissFailed={jobs.dismissFailed}
|
||||
newlyCompletedItems={jobs.newlyCompleted}
|
||||
galleryItems={gallery.items}
|
||||
newlyCompletedTokens={newlyCompletedTokens}
|
||||
hasMore={gallery.hasMore}
|
||||
isLoading={gallery.isLoading}
|
||||
isInitialLoading={gallery.isInitialLoading}
|
||||
onLoadMore={gallery.loadMore}
|
||||
onGalleryItemClick={lightbox.handleGalleryItemClick}
|
||||
/>
|
||||
}
|
||||
promptBox={
|
||||
<div
|
||||
ref={promptBoxRef}
|
||||
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
|
||||
style={{
|
||||
animationDelay: "150ms",
|
||||
left: "var(--ac-sidebar-offset, 0px)",
|
||||
}}
|
||||
>
|
||||
<PromptBox
|
||||
prompt={prompt}
|
||||
onPromptChange={setPrompt}
|
||||
onSubmit={handleGenerate}
|
||||
isSubmitting={isGenerating}
|
||||
disabled={!canGenerate}
|
||||
credits={estimatedCredits}
|
||||
placeholder={placeholder}
|
||||
supportsImagePrompts={supportsImage}
|
||||
maxImagePromptCount={1}
|
||||
referenceImages={referenceImages}
|
||||
onReferenceImagesChange={setReferenceImages}
|
||||
onPickFromLibrary={() => setIsImagePickerOpen(true)}
|
||||
mediaReferenceRow={meshInputsRow}
|
||||
modelSelector={
|
||||
<Tooltip content="Model" position="top" className="z-50" closeOnClick>
|
||||
<PopoverMenu
|
||||
items={modelItems}
|
||||
onSelect={handleModelChange}
|
||||
mode="toggle"
|
||||
panelTitle="Select Model"
|
||||
panelClassName="w-[360px]"
|
||||
richList
|
||||
triggerIcon={
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
|
||||
alt=""
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
}
|
||||
leftToolbar={optionControls}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
modals={
|
||||
<>
|
||||
<GalleryModal
|
||||
mode="select"
|
||||
isOpen={isImagePickerOpen}
|
||||
onClose={() => setIsImagePickerOpen(false)}
|
||||
selectedItemIds={pickerSelectedIds}
|
||||
onSelectItem={(id) => setPickerSelectedIds([id])}
|
||||
maxSelections={1}
|
||||
onUseSelected={handleLibraryImageSelect}
|
||||
forceFilter="image"
|
||||
hideFilter
|
||||
/>
|
||||
<Lightbox
|
||||
isOpen={lightbox.lightboxOpen}
|
||||
onClose={lightbox.closeLightbox}
|
||||
mediaToken={lightbox.lightboxItem?.id}
|
||||
cdnUrl={lightbox.lightboxItem?.fullImage}
|
||||
mediaClass={lightbox.lightboxItem?.mediaClass}
|
||||
batchImageToken={lightbox.lightboxItem?.batchImageToken}
|
||||
showBatchCarousel={false}
|
||||
onNavigatePrev={lightbox.navigatePrev}
|
||||
onNavigateNext={lightbox.navigateNext}
|
||||
onDeleted={gallery.removeItem}
|
||||
/>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import { JobsApi, MediaFilesApi, OmniGenApi } from "@storyteller/api";
|
||||
import type { OmniGenMeshRequest } from "@storyteller/api";
|
||||
import type { GeneratedAsset } from "./create-object-store";
|
||||
|
||||
// ── Request params ───────────────────────────────────────────────────────
|
||||
|
||||
export interface GenerateMeshParams {
|
||||
model: string;
|
||||
prompt?: string;
|
||||
referenceImageMediaTokens?: string[];
|
||||
frontImageMediaToken?: string;
|
||||
backImageMediaToken?: string;
|
||||
leftImageMediaToken?: string;
|
||||
rightImageMediaToken?: string;
|
||||
inputMeshMediaToken?: string;
|
||||
meshOutputType?: string;
|
||||
polygonType?: string;
|
||||
faceCount?: number;
|
||||
enablePbr?: boolean;
|
||||
enableTexture?: boolean;
|
||||
textureQuality?: string;
|
||||
geometryQuality?: string;
|
||||
}
|
||||
|
||||
// ── Enqueue generation ───────────────────────────────────────────────────
|
||||
|
||||
export async function enqueueMeshGeneration(params: GenerateMeshParams): Promise<{
|
||||
success: boolean;
|
||||
jobToken?: string;
|
||||
error?: string;
|
||||
errorCode?: number;
|
||||
}> {
|
||||
const body: OmniGenMeshRequest = {
|
||||
model: params.model,
|
||||
idempotency_token: crypto.randomUUID(),
|
||||
prompt: params.prompt ?? null,
|
||||
reference_image_media_tokens: params.referenceImageMediaTokens?.length
|
||||
? params.referenceImageMediaTokens
|
||||
: null,
|
||||
front_image_media_token: params.frontImageMediaToken ?? null,
|
||||
back_image_media_token: params.backImageMediaToken ?? null,
|
||||
left_image_media_token: params.leftImageMediaToken ?? null,
|
||||
right_image_media_token: params.rightImageMediaToken ?? null,
|
||||
input_mesh_media_token: params.inputMeshMediaToken ?? null,
|
||||
mesh_output_type: params.meshOutputType ?? null,
|
||||
polygon_type: params.polygonType ?? null,
|
||||
face_count: params.faceCount ?? null,
|
||||
enable_pbr: params.enablePbr ?? null,
|
||||
enable_texture: params.enableTexture ?? null,
|
||||
texture_quality: params.textureQuality ?? null,
|
||||
geometry_quality: params.geometryQuality ?? null,
|
||||
};
|
||||
|
||||
try {
|
||||
const api = new OmniGenApi();
|
||||
const response = await api.generateMesh(body);
|
||||
if (response.success && response.inference_job_token) {
|
||||
return { success: true, jobToken: response.inference_job_token };
|
||||
}
|
||||
return { success: false, error: "Generation failed" };
|
||||
} catch (err: any) {
|
||||
return {
|
||||
success: false,
|
||||
error: err.message ?? "Request failed",
|
||||
errorCode: parseHttpStatusCode(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Pull the HTTP status out of an ApiManager error (thrown as
|
||||
// `Error("HTTP error! status: 402")`) so callers can special-case 402.
|
||||
function parseHttpStatusCode(err: unknown): number | undefined {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const match = /status:\s*(\d+)/.exec(message);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
// ── Poll for completion ──────────────────────────────────────────────────
|
||||
|
||||
async function pollJobResult(
|
||||
jobToken: string,
|
||||
): Promise<{ status: "pending" | "complete" | "failed"; assets: GeneratedAsset[]; error?: string }> {
|
||||
const jobsApi = new JobsApi();
|
||||
const response = await jobsApi.GetJobByToken({ token: jobToken });
|
||||
|
||||
if (!response.success || !response.data) {
|
||||
return { status: "pending", assets: [] };
|
||||
}
|
||||
|
||||
const state = response.data;
|
||||
const statusStr = state.status?.status?.toLowerCase() ?? "";
|
||||
|
||||
if (statusStr === "complete_success" || statusStr === "complete") {
|
||||
const result = state.maybe_result as Record<string, unknown> | undefined;
|
||||
const mediaLinks = (result as any)?.media_links;
|
||||
const entityToken = (result as any)?.entity_token as string | undefined;
|
||||
|
||||
if (mediaLinks?.cdn_url) {
|
||||
const asset: GeneratedAsset = {
|
||||
media_token: entityToken ?? jobToken,
|
||||
cdn_url: mediaLinks.cdn_url,
|
||||
maybe_thumbnail_template: mediaLinks.maybe_thumbnail_template,
|
||||
};
|
||||
return { status: "complete", assets: [asset] };
|
||||
}
|
||||
|
||||
// Rare: no media link on the result payload. Confirm the media file exists.
|
||||
if (entityToken) {
|
||||
try {
|
||||
const mediaApi = new MediaFilesApi();
|
||||
const mediaFile = await mediaApi.GetMediaFileByToken({
|
||||
mediaFileToken: entityToken,
|
||||
});
|
||||
const cdnUrl = (mediaFile.data as any)?.media_links?.cdn_url;
|
||||
if (cdnUrl) {
|
||||
return {
|
||||
status: "complete",
|
||||
assets: [{ media_token: entityToken, cdn_url: cdnUrl }],
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
|
||||
return { status: "complete", assets: [] };
|
||||
}
|
||||
|
||||
if (
|
||||
statusStr.includes("fail") ||
|
||||
statusStr.includes("error") ||
|
||||
statusStr === "dead"
|
||||
) {
|
||||
return {
|
||||
status: "failed",
|
||||
assets: [],
|
||||
error:
|
||||
state.status?.maybe_failure_message ??
|
||||
state.status?.maybe_extra_status_description ??
|
||||
"Generation failed",
|
||||
};
|
||||
}
|
||||
|
||||
return { status: "pending", assets: [] };
|
||||
}
|
||||
|
||||
// ── Polling controller ───────────────────────────────────────────────────
|
||||
|
||||
// 3D generations can run a while, so allow a longer window than images.
|
||||
export function startPolling(
|
||||
jobToken: string,
|
||||
onComplete: (assets: GeneratedAsset[]) => void,
|
||||
onError: (reason: string) => void,
|
||||
intervalMs = 4000,
|
||||
maxAttempts = 150,
|
||||
): () => void {
|
||||
let attempts = 0;
|
||||
let stopped = false;
|
||||
|
||||
const poll = async () => {
|
||||
if (stopped) return;
|
||||
attempts++;
|
||||
|
||||
try {
|
||||
const result = await pollJobResult(jobToken);
|
||||
if (stopped) return;
|
||||
|
||||
if (result.status === "complete") {
|
||||
onComplete(result.assets);
|
||||
return;
|
||||
}
|
||||
if (result.status === "failed") {
|
||||
onError(result.error ?? "Generation failed");
|
||||
return;
|
||||
}
|
||||
if (attempts >= maxAttempts) {
|
||||
onError("Generation timed out");
|
||||
return;
|
||||
}
|
||||
|
||||
setTimeout(poll, intervalMs);
|
||||
} catch {
|
||||
if (!stopped && attempts < maxAttempts) {
|
||||
setTimeout(poll, intervalMs * 2);
|
||||
} else if (!stopped) {
|
||||
onError("Network error during polling");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
setTimeout(poll, 2000);
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from "./create-object";
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
import { useRef, useState } from "react";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { faPlus, faSpinnerThird, faXmark } from "@fortawesome/pro-solid-svg-icons";
|
||||
import { MediaUploadApi, EIntermediateFile } from "@storyteller/api";
|
||||
import type { RefVideoAsset } from "../create-world-store";
|
||||
|
||||
// A single optional reference video for splat generation. Uploads via the
|
||||
// generic new-video endpoint and stores the returned media token.
|
||||
|
||||
const SLOT_CLASS =
|
||||
"flex aspect-square w-14 items-center justify-center overflow-hidden rounded-lg border-2 border-dashed border-white/25 bg-white/5 transition-all hover:border-white/40 hover:bg-white/10";
|
||||
|
||||
export function ReferenceVideoSlot({
|
||||
video,
|
||||
onChange,
|
||||
}: {
|
||||
video?: RefVideoAsset;
|
||||
onChange: (video?: RefVideoAsset) => void;
|
||||
}) {
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
|
||||
const handleUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.target.files?.[0];
|
||||
if (!file) return;
|
||||
setUploading(true);
|
||||
const previewUrl = URL.createObjectURL(file);
|
||||
try {
|
||||
const api = new MediaUploadApi();
|
||||
const response = await api.UploadNewVideo({
|
||||
blob: file,
|
||||
fileName: file.name || `reference-video-${Date.now()}`,
|
||||
uuid: crypto.randomUUID(),
|
||||
maybe_title: "splat_reference_video",
|
||||
is_intermediate_system_file: EIntermediateFile.false,
|
||||
});
|
||||
if (response?.success && response.data) {
|
||||
onChange({
|
||||
id: Math.random().toString(36).substring(7),
|
||||
previewUrl,
|
||||
mediaToken: response.data,
|
||||
});
|
||||
} else {
|
||||
URL.revokeObjectURL(previewUrl);
|
||||
}
|
||||
} catch {
|
||||
URL.revokeObjectURL(previewUrl);
|
||||
} finally {
|
||||
setUploading(false);
|
||||
if (inputRef.current) inputRef.current.value = "";
|
||||
}
|
||||
};
|
||||
|
||||
const handleRemove = () => {
|
||||
if (video?.previewUrl) URL.revokeObjectURL(video.previewUrl);
|
||||
onChange(undefined);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="glass flex items-center gap-3 rounded-2xl px-3 py-2">
|
||||
<div className="flex flex-col">
|
||||
<span className="text-sm font-medium text-white/90">Reference video</span>
|
||||
<span className="text-[13px] text-white/60">Guide the world (optional)</span>
|
||||
</div>
|
||||
<input
|
||||
type="file"
|
||||
ref={inputRef}
|
||||
className="hidden"
|
||||
accept="video/*"
|
||||
onChange={handleUpload}
|
||||
/>
|
||||
{video ? (
|
||||
<div className="group relative aspect-square w-14 overflow-hidden rounded-lg border-2 border-white/30">
|
||||
<video
|
||||
src={video.previewUrl}
|
||||
muted
|
||||
preload="metadata"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
<button
|
||||
onClick={handleRemove}
|
||||
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/50 text-white backdrop-blur-md transition-colors hover:bg-black"
|
||||
>
|
||||
<FontAwesomeIcon icon={faXmark} className="h-2.5 w-2.5" />
|
||||
</button>
|
||||
</div>
|
||||
) : uploading ? (
|
||||
<div className={SLOT_CLASS}>
|
||||
<FontAwesomeIcon icon={faSpinnerThird} spin className="h-5 w-5 text-white" />
|
||||
</div>
|
||||
) : (
|
||||
<button onClick={() => inputRef.current?.click()} className={SLOT_CLASS}>
|
||||
<FontAwesomeIcon icon={faPlus} className="text-xl text-white/80" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import { create } from "zustand";
|
||||
import { persist } from "zustand/middleware";
|
||||
import type { RecreatePayload } from "../../lib/recreate";
|
||||
import type { RefImage } from "../../components/prompt-box";
|
||||
|
||||
// A generated splat (3D world). `cdn_url` here is the .spz file.
|
||||
export interface GeneratedAsset {
|
||||
media_token: string;
|
||||
cdn_url: string;
|
||||
maybe_thumbnail_template?: string;
|
||||
}
|
||||
|
||||
export type SplatBatch = {
|
||||
id: string;
|
||||
prompt: string;
|
||||
status: "pending" | "complete" | "failed";
|
||||
assets: GeneratedAsset[];
|
||||
createdAt: number;
|
||||
modelLabel: string;
|
||||
jobToken?: string;
|
||||
failureReason?: string;
|
||||
};
|
||||
|
||||
export type SplatUiState = {
|
||||
selectedModelId: string | null;
|
||||
prompt: string;
|
||||
isPanoramic: boolean | undefined;
|
||||
disableRecaption: boolean | undefined;
|
||||
};
|
||||
|
||||
// A stored reference video: media token + a local object-URL for preview. Not
|
||||
// persisted (File handles / blob URLs aren't serializable).
|
||||
export type RefVideoAsset = {
|
||||
id: string;
|
||||
previewUrl: string;
|
||||
mediaToken: string;
|
||||
};
|
||||
|
||||
type CreateWorldState = {
|
||||
batches: SplatBatch[];
|
||||
ui: SplatUiState;
|
||||
referenceImages: RefImage[];
|
||||
referenceVideo: RefVideoAsset | undefined;
|
||||
pendingRecreate: RecreatePayload | null;
|
||||
setUi: (patch: Partial<SplatUiState>) => void;
|
||||
setReferenceImages: (images: RefImage[]) => void;
|
||||
setReferenceVideo: (video: RefVideoAsset | undefined) => void;
|
||||
setPendingRecreate: (payload: RecreatePayload | null) => void;
|
||||
consumePendingRecreate: () => RecreatePayload | null;
|
||||
startBatch: (prompt: string, modelLabel: string) => string;
|
||||
setBatchJobToken: (batchId: string, jobToken: string) => void;
|
||||
completeBatch: (batchId: string, assets: GeneratedAsset[]) => void;
|
||||
failBatch: (batchId: string, reason?: string) => void;
|
||||
dismissBatch: (id: string) => void;
|
||||
reset: () => void;
|
||||
};
|
||||
|
||||
const DEFAULT_UI: SplatUiState = {
|
||||
selectedModelId: null,
|
||||
prompt: "",
|
||||
isPanoramic: undefined,
|
||||
disableRecaption: undefined,
|
||||
};
|
||||
|
||||
export const useCreateWorldStore = create<CreateWorldState>()(
|
||||
persist(
|
||||
(set, get) => ({
|
||||
batches: [],
|
||||
ui: { ...DEFAULT_UI },
|
||||
referenceImages: [],
|
||||
referenceVideo: undefined,
|
||||
pendingRecreate: null,
|
||||
|
||||
setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })),
|
||||
|
||||
setReferenceImages: (images) => set({ referenceImages: images }),
|
||||
|
||||
setReferenceVideo: (video) => set({ referenceVideo: video }),
|
||||
|
||||
setPendingRecreate: (payload) => set({ pendingRecreate: payload }),
|
||||
|
||||
consumePendingRecreate: () => {
|
||||
const payload = get().pendingRecreate;
|
||||
if (payload) set({ pendingRecreate: null });
|
||||
return payload;
|
||||
},
|
||||
|
||||
startBatch: (prompt, modelLabel) => {
|
||||
const id = crypto.randomUUID();
|
||||
const batch: SplatBatch = {
|
||||
id,
|
||||
prompt,
|
||||
status: "pending",
|
||||
assets: [],
|
||||
createdAt: Date.now(),
|
||||
modelLabel,
|
||||
};
|
||||
set((s) => ({ batches: [...s.batches, batch] }));
|
||||
return id;
|
||||
},
|
||||
|
||||
setBatchJobToken: (batchId, jobToken) => {
|
||||
set((s) => ({
|
||||
batches: s.batches.map((b) =>
|
||||
b.id === batchId ? { ...b, jobToken } : b,
|
||||
),
|
||||
}));
|
||||
},
|
||||
|
||||
completeBatch: (batchId, assets) => {
|
||||
set((s) => ({
|
||||
batches: s.batches.map((b) =>
|
||||
b.id === batchId
|
||||
? { ...b, status: "complete" as const, assets }
|
||||
: b,
|
||||
),
|
||||
}));
|
||||
},
|
||||
|
||||
failBatch: (batchId, reason) => {
|
||||
set((s) => ({
|
||||
batches: s.batches.map((b) =>
|
||||
b.id === batchId
|
||||
? { ...b, status: "failed" as const, failureReason: reason }
|
||||
: b,
|
||||
),
|
||||
}));
|
||||
},
|
||||
|
||||
dismissBatch: (id) => {
|
||||
set((s) => ({ batches: s.batches.filter((b) => b.id !== id) }));
|
||||
},
|
||||
|
||||
reset: () => set({ batches: [] }),
|
||||
}),
|
||||
{
|
||||
name: "artcraft-world-prompt",
|
||||
partialize: (state) => ({ ui: state.ui }),
|
||||
merge: (persisted, current) => {
|
||||
const persistedUi = (persisted as { ui?: Partial<SplatUiState> } | null)
|
||||
?.ui;
|
||||
return { ...current, ui: { ...current.ui, ...(persistedUi ?? {}) } };
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
@@ -0,0 +1,537 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { FilterMediaClasses } from "@storyteller/api";
|
||||
import type { OmniGenSplatModelInfo } from "@storyteller/api";
|
||||
import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
import { Button, ToggleButton } from "@storyteller/ui-button";
|
||||
import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal";
|
||||
import {
|
||||
faSparkles,
|
||||
faPanorama,
|
||||
faWandMagicSparkles,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
PromptBox,
|
||||
ImagePromptRow,
|
||||
MobilePromptForm,
|
||||
MobileSelectField,
|
||||
type RefImage,
|
||||
} from "../../components/prompt-box";
|
||||
import {
|
||||
GenerationGallery,
|
||||
useGalleryData,
|
||||
useGenerationJobs,
|
||||
useAuthCheck,
|
||||
usePromptHeight,
|
||||
useLightboxNav,
|
||||
CreateMediaPageShell,
|
||||
} from "../../components/generation-gallery";
|
||||
import { Lightbox } from "../../components/lightbox/lightbox";
|
||||
import { useCreateWorldStore } from "./create-world-store";
|
||||
import { enqueueSplatGeneration, startPolling } from "./generate-world-api";
|
||||
import { ReferenceVideoSlot } from "./components/ReferenceVideoSlot";
|
||||
import { useSplatCostEstimate } from "../../lib/cost-estimate-api";
|
||||
import {
|
||||
useOmniGenSplatModels,
|
||||
OMNI_GENERATE_OUTAGE_MESSAGE,
|
||||
} from "@storyteller/omni-gen";
|
||||
import {
|
||||
getCreatorIconPathForModelId,
|
||||
getModelDescription,
|
||||
getModelInfo,
|
||||
} from "@storyteller/model-list";
|
||||
import { toast } from "../../components/toast/toast";
|
||||
import { useSignupCta } from "../../components/signup-cta-modal";
|
||||
import { useInsufficientCredits } from "../../components/insufficient-credits-modal";
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_MODEL_ID = "marble_1p1";
|
||||
const DIMENSIONAL_FILTER = [FilterMediaClasses.DIMENSIONAL];
|
||||
|
||||
let _modelLookup = new Map<string, OmniGenSplatModelInfo>();
|
||||
|
||||
function buildModelPopoverItems(
|
||||
models: OmniGenSplatModelInfo[],
|
||||
selectedId: string,
|
||||
): PopoverItem[] {
|
||||
_modelLookup = new Map(models.map((m) => [m.model, m]));
|
||||
return models.map((model) => ({
|
||||
label: model.full_name || model.model,
|
||||
selected: model.model === selectedId,
|
||||
description: getModelDescription(model.model, model.extra_info_short),
|
||||
info: getModelInfo(model.model, model.extra_info) || undefined,
|
||||
icon: (
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(model.model)}
|
||||
alt={`${model.model} logo`}
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
),
|
||||
action: model.model,
|
||||
}));
|
||||
}
|
||||
|
||||
// ── Component ────────────────────────────────────────────────────────────
|
||||
|
||||
export default function CreateWorld() {
|
||||
const { user, authChecked } = useAuthCheck();
|
||||
const { loggedIn, openSignupCta } = useSignupCta();
|
||||
const openInsufficientCredits = useInsufficientCredits();
|
||||
const { promptBoxRef, promptHeight } = usePromptHeight();
|
||||
|
||||
const { models: apiModels } = useOmniGenSplatModels();
|
||||
|
||||
const ui = useCreateWorldStore((s) => s.ui);
|
||||
const setUi = useCreateWorldStore((s) => s.setUi);
|
||||
|
||||
const selectedModel = useMemo((): OmniGenSplatModelInfo | undefined => {
|
||||
if (!apiModels.length) return undefined;
|
||||
if (ui.selectedModelId) {
|
||||
return (
|
||||
apiModels.find((m) => m.model === ui.selectedModelId) ??
|
||||
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ??
|
||||
apiModels[0]
|
||||
);
|
||||
}
|
||||
return apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0];
|
||||
}, [apiModels, ui.selectedModelId]);
|
||||
|
||||
const prompt = ui.prompt;
|
||||
const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]);
|
||||
|
||||
const supportsText = !!selectedModel?.text_prompt_supported;
|
||||
const supportsImage = !!selectedModel?.image_references_supported;
|
||||
const maxImageRefs = selectedModel?.image_references_max ?? 4;
|
||||
const supportsVideo = !!selectedModel?.video_reference_supported;
|
||||
const supportsPanorama = !!selectedModel?.panorama_supported;
|
||||
const supportsDisableRecaption = !!selectedModel?.disable_recaption_supported;
|
||||
|
||||
const isPanoramic = supportsPanorama ? (ui.isPanoramic ?? false) : false;
|
||||
const disableRecaption = supportsDisableRecaption
|
||||
? (ui.disableRecaption ?? false)
|
||||
: false;
|
||||
|
||||
const [isGenerating, setIsGenerating] = useState(false);
|
||||
const referenceImages = useCreateWorldStore((s) => s.referenceImages);
|
||||
const setReferenceImages = useCreateWorldStore((s) => s.setReferenceImages);
|
||||
const referenceVideo = useCreateWorldStore((s) => s.referenceVideo);
|
||||
const setReferenceVideo = useCreateWorldStore((s) => s.setReferenceVideo);
|
||||
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
|
||||
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isImagePickerOpen) setPickerSelectedIds([]);
|
||||
}, [isImagePickerOpen]);
|
||||
|
||||
const imagePickerMax = Math.max(1, maxImageRefs - referenceImages.length);
|
||||
const handlePickerSelect = useCallback(
|
||||
(id: string) => {
|
||||
setPickerSelectedIds((prev) => {
|
||||
if (prev.includes(id)) return prev.filter((x) => x !== id);
|
||||
if (prev.length >= imagePickerMax) {
|
||||
return imagePickerMax === 1 ? [id] : prev;
|
||||
}
|
||||
return [...prev, id];
|
||||
});
|
||||
},
|
||||
[imagePickerMax],
|
||||
);
|
||||
|
||||
// Batch store
|
||||
const startBatch = useCreateWorldStore((s) => s.startBatch);
|
||||
const setBatchJobToken = useCreateWorldStore((s) => s.setBatchJobToken);
|
||||
const completeBatch = useCreateWorldStore((s) => s.completeBatch);
|
||||
const failBatch = useCreateWorldStore((s) => s.failBatch);
|
||||
const dismissBatch = useCreateWorldStore((s) => s.dismissBatch);
|
||||
const pollingCleanupsRef = useRef<Map<string, () => void>>(new Map());
|
||||
|
||||
// Jobs + gallery (only splats, not mesh objects)
|
||||
const splatModelIds = useMemo(
|
||||
() => apiModels.map((m) => m.model),
|
||||
[apiModels],
|
||||
);
|
||||
const jobs = useGenerationJobs({ mediaType: "splat", enabled: !!user });
|
||||
const gallery = useGalleryData({
|
||||
username: user?.username ?? null,
|
||||
filterMediaClasses: DIMENSIONAL_FILTER,
|
||||
filterModelIds: splatModelIds,
|
||||
excludeUploads: true,
|
||||
});
|
||||
|
||||
const newlyCompletedTokens = useMemo(
|
||||
() => new Set(jobs.newlyCompleted.map((i) => i.id)),
|
||||
[jobs.newlyCompleted],
|
||||
);
|
||||
|
||||
const flatItems = useMemo(() => {
|
||||
const filtered = gallery.items.filter(
|
||||
(i) => !newlyCompletedTokens.has(i.id),
|
||||
);
|
||||
return [...jobs.newlyCompleted, ...filtered];
|
||||
}, [jobs.newlyCompleted, gallery.items, newlyCompletedTokens]);
|
||||
|
||||
const lightbox = useLightboxNav(flatItems);
|
||||
|
||||
const estimatedCredits = useSplatCostEstimate({
|
||||
model: selectedModel?.model ?? "",
|
||||
referenceImageCount: referenceImages.length,
|
||||
hasReferenceVideo: !!referenceVideo,
|
||||
isPanoramic: supportsPanorama ? isPanoramic : undefined,
|
||||
disableRecaption: supportsDisableRecaption ? disableRecaption : undefined,
|
||||
});
|
||||
|
||||
const modelItems = useMemo(
|
||||
() => buildModelPopoverItems(apiModels, selectedModel?.model ?? ""),
|
||||
[apiModels, selectedModel?.model],
|
||||
);
|
||||
|
||||
const hasContent =
|
||||
jobs.inProgress.length > 0 ||
|
||||
jobs.failed.length > 0 ||
|
||||
jobs.newlyCompleted.length > 0 ||
|
||||
gallery.items.length > 0 ||
|
||||
gallery.isInitialLoading;
|
||||
|
||||
const canGenerate =
|
||||
!!selectedModel &&
|
||||
!isGenerating &&
|
||||
(prompt.trim().length > 0 || referenceImages.length > 0 || !!referenceVideo);
|
||||
|
||||
// ── Effects ──────────────────────────────────────────────────────────────
|
||||
|
||||
useEffect(() => {
|
||||
const cleanups = pollingCleanupsRef.current;
|
||||
const pendingBatches = useCreateWorldStore
|
||||
.getState()
|
||||
.batches.filter((b) => b.status === "pending" && b.jobToken);
|
||||
|
||||
for (const batch of pendingBatches) {
|
||||
if (cleanups.has(batch.id)) continue;
|
||||
const stop = startPolling(
|
||||
batch.jobToken!,
|
||||
(assets) => {
|
||||
completeBatch(batch.id, assets);
|
||||
cleanups.delete(batch.id);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
(reason) => {
|
||||
failBatch(batch.id, reason);
|
||||
cleanups.delete(batch.id);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
);
|
||||
cleanups.set(batch.id, stop);
|
||||
}
|
||||
|
||||
return () => {
|
||||
cleanups.forEach((stop) => stop());
|
||||
cleanups.clear();
|
||||
};
|
||||
}, [completeBatch, failBatch]);
|
||||
|
||||
// ── Handlers ─────────────────────────────────────────────────────────────
|
||||
|
||||
const handleModelChange = useCallback(
|
||||
(item: PopoverItem) => {
|
||||
const model = item.action ? _modelLookup.get(item.action) : undefined;
|
||||
if (!model) return;
|
||||
setUi({ selectedModelId: model.model });
|
||||
},
|
||||
[setUi],
|
||||
);
|
||||
|
||||
const handleLibraryImageSelect = useCallback(
|
||||
(items: GalleryItem[]) => {
|
||||
const availableSlots = Math.max(0, maxImageRefs - referenceImages.length);
|
||||
const newImages: RefImage[] = items
|
||||
.slice(0, availableSlots)
|
||||
.map((item) => ({
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: item.thumbnail || item.fullImage || "",
|
||||
fullUrl: item.fullImage || undefined,
|
||||
file: new File([], "library-image"),
|
||||
mediaToken: item.id,
|
||||
}));
|
||||
setReferenceImages([...referenceImages, ...newImages]);
|
||||
setIsImagePickerOpen(false);
|
||||
},
|
||||
[maxImageRefs, referenceImages, setReferenceImages],
|
||||
);
|
||||
|
||||
const handleGenerate = useCallback(async () => {
|
||||
if (!loggedIn) {
|
||||
openSignupCta();
|
||||
return;
|
||||
}
|
||||
if (!canGenerate || !selectedModel) return;
|
||||
|
||||
setIsGenerating(true);
|
||||
const batchId = startBatch(
|
||||
prompt || selectedModel.full_name || selectedModel.model,
|
||||
selectedModel.full_name ?? selectedModel.model,
|
||||
);
|
||||
|
||||
try {
|
||||
const referenceImageMediaTokens = supportsImage
|
||||
? referenceImages
|
||||
.map((img) => img.mediaToken)
|
||||
.filter((t): t is string => typeof t === "string" && t.length > 0)
|
||||
: undefined;
|
||||
|
||||
const result = await enqueueSplatGeneration({
|
||||
model: selectedModel.model,
|
||||
prompt: supportsText && prompt.trim() ? prompt.trim() : undefined,
|
||||
referenceImageMediaTokens: referenceImageMediaTokens?.length
|
||||
? referenceImageMediaTokens
|
||||
: undefined,
|
||||
referenceVideoMediaToken: supportsVideo
|
||||
? referenceVideo?.mediaToken
|
||||
: undefined,
|
||||
isPanoramic: supportsPanorama ? isPanoramic : undefined,
|
||||
disableRecaption: supportsDisableRecaption ? disableRecaption : undefined,
|
||||
});
|
||||
|
||||
if (!result.success || !result.jobToken) {
|
||||
if (result.errorCode === 402) {
|
||||
dismissBatch(batchId);
|
||||
openInsufficientCredits();
|
||||
} else {
|
||||
if (result.errorCode != null && result.errorCode >= 500) {
|
||||
toast.error(OMNI_GENERATE_OUTAGE_MESSAGE);
|
||||
}
|
||||
failBatch(batchId, result.error ?? "Failed to start generation");
|
||||
}
|
||||
setIsGenerating(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setBatchJobToken(batchId, result.jobToken);
|
||||
window.dispatchEvent(new Event("credits-change"));
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
|
||||
const stopPolling = startPolling(
|
||||
result.jobToken,
|
||||
(assets) => {
|
||||
completeBatch(batchId, assets);
|
||||
pollingCleanupsRef.current.delete(batchId);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
(reason) => {
|
||||
failBatch(batchId, reason);
|
||||
pollingCleanupsRef.current.delete(batchId);
|
||||
window.dispatchEvent(new Event("task-queue-update"));
|
||||
},
|
||||
);
|
||||
pollingCleanupsRef.current.set(batchId, stopPolling);
|
||||
} catch {
|
||||
failBatch(batchId, "Network error - please try again");
|
||||
} finally {
|
||||
setIsGenerating(false);
|
||||
}
|
||||
}, [
|
||||
loggedIn,
|
||||
openSignupCta,
|
||||
openInsufficientCredits,
|
||||
canGenerate,
|
||||
selectedModel,
|
||||
prompt,
|
||||
supportsText,
|
||||
supportsImage,
|
||||
supportsVideo,
|
||||
supportsPanorama,
|
||||
supportsDisableRecaption,
|
||||
referenceImages,
|
||||
referenceVideo,
|
||||
isPanoramic,
|
||||
disableRecaption,
|
||||
startBatch,
|
||||
setBatchJobToken,
|
||||
completeBatch,
|
||||
failBatch,
|
||||
dismissBatch,
|
||||
]);
|
||||
|
||||
// ── Option controls (shared desktop + mobile) ──────────────────────────
|
||||
|
||||
const optionControls = (
|
||||
<>
|
||||
{supportsPanorama && (
|
||||
<Tooltip content="360° panorama input" position="top" closeOnClick>
|
||||
<ToggleButton
|
||||
isActive={isPanoramic}
|
||||
icon={faPanorama}
|
||||
activeIcon={faPanorama}
|
||||
label={isPanoramic ? "Panorama" : "Panorama off"}
|
||||
onClick={() => setUi({ isPanoramic: !isPanoramic })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
{supportsDisableRecaption && (
|
||||
<Tooltip content="Use prompt exactly (skip recaption)" position="top" closeOnClick>
|
||||
<ToggleButton
|
||||
isActive={disableRecaption}
|
||||
icon={faWandMagicSparkles}
|
||||
activeIcon={faWandMagicSparkles}
|
||||
label={disableRecaption ? "No recaption" : "Recaption"}
|
||||
onClick={() => setUi({ disableRecaption: !disableRecaption })}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const videoRow = supportsVideo ? (
|
||||
<ReferenceVideoSlot video={referenceVideo} onChange={setReferenceVideo} />
|
||||
) : undefined;
|
||||
|
||||
// ── Mobile form ─────────────────────────────────────────────────────────
|
||||
|
||||
const mobileForm = (
|
||||
<MobilePromptForm
|
||||
prompt={prompt}
|
||||
onPromptChange={setPrompt}
|
||||
onSubmit={handleGenerate}
|
||||
isSubmitting={isGenerating}
|
||||
credits={estimatedCredits}
|
||||
placeholder="Describe the 3D world you want..."
|
||||
autoAdvance={loggedIn && canGenerate}
|
||||
modelField={
|
||||
<MobileSelectField
|
||||
label="Model"
|
||||
title="Select Model"
|
||||
items={modelItems}
|
||||
onSelect={handleModelChange}
|
||||
/>
|
||||
}
|
||||
frames={
|
||||
supportsImage ? (
|
||||
<ImagePromptRow
|
||||
maxImagePromptCount={maxImageRefs}
|
||||
referenceImages={referenceImages}
|
||||
setReferenceImages={setReferenceImages}
|
||||
onPickFromLibrary={() => setIsImagePickerOpen(true)}
|
||||
/>
|
||||
) : undefined
|
||||
}
|
||||
settingsFields={videoRow}
|
||||
/>
|
||||
);
|
||||
|
||||
// ── Render ────────────────────────────────────────────────────────────
|
||||
|
||||
return (
|
||||
<CreateMediaPageShell
|
||||
title="3D World - ArtCraft"
|
||||
description="Generate explorable 3D worlds with ArtCraft"
|
||||
authChecked={authChecked}
|
||||
hasContent={hasContent}
|
||||
emptyStateTitle="Create 3D World"
|
||||
emptyStateSubtitle="Turn a prompt or image into an explorable world."
|
||||
emptyStateCta={
|
||||
loggedIn ? undefined : (
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={openSignupCta}
|
||||
icon={faSparkles}
|
||||
className="h-12 px-6 text-base font-semibold rounded-full"
|
||||
>
|
||||
Sign up to create
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
bottomOffset={promptHeight + 24}
|
||||
modelItems={modelItems}
|
||||
onModelChange={handleModelChange}
|
||||
promptForm={mobileForm}
|
||||
gridContent={
|
||||
<GenerationGallery
|
||||
inProgressJobs={jobs.inProgress}
|
||||
failedJobs={jobs.failed}
|
||||
onDismissFailed={jobs.dismissFailed}
|
||||
newlyCompletedItems={jobs.newlyCompleted}
|
||||
galleryItems={gallery.items}
|
||||
newlyCompletedTokens={newlyCompletedTokens}
|
||||
hasMore={gallery.hasMore}
|
||||
isLoading={gallery.isLoading}
|
||||
isInitialLoading={gallery.isInitialLoading}
|
||||
onLoadMore={gallery.loadMore}
|
||||
onGalleryItemClick={lightbox.handleGalleryItemClick}
|
||||
/>
|
||||
}
|
||||
promptBox={
|
||||
<div
|
||||
ref={promptBoxRef}
|
||||
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
|
||||
style={{
|
||||
animationDelay: "150ms",
|
||||
left: "var(--ac-sidebar-offset, 0px)",
|
||||
}}
|
||||
>
|
||||
<PromptBox
|
||||
prompt={prompt}
|
||||
onPromptChange={setPrompt}
|
||||
onSubmit={handleGenerate}
|
||||
isSubmitting={isGenerating}
|
||||
disabled={!canGenerate}
|
||||
credits={estimatedCredits}
|
||||
placeholder="Describe the 3D world you want..."
|
||||
supportsImagePrompts={supportsImage}
|
||||
maxImagePromptCount={maxImageRefs}
|
||||
referenceImages={referenceImages}
|
||||
onReferenceImagesChange={setReferenceImages}
|
||||
onPickFromLibrary={() => setIsImagePickerOpen(true)}
|
||||
mediaReferenceRow={videoRow}
|
||||
modelSelector={
|
||||
<Tooltip content="Model" position="top" className="z-50" closeOnClick>
|
||||
<PopoverMenu
|
||||
items={modelItems}
|
||||
onSelect={handleModelChange}
|
||||
mode="toggle"
|
||||
panelTitle="Select Model"
|
||||
panelClassName="w-[360px]"
|
||||
richList
|
||||
triggerIcon={
|
||||
<img
|
||||
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
|
||||
alt=""
|
||||
className="h-4 w-4 icon-auto-contrast"
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
}
|
||||
leftToolbar={optionControls}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
modals={
|
||||
<>
|
||||
<GalleryModal
|
||||
mode="select"
|
||||
isOpen={isImagePickerOpen}
|
||||
onClose={() => setIsImagePickerOpen(false)}
|
||||
selectedItemIds={pickerSelectedIds}
|
||||
onSelectItem={handlePickerSelect}
|
||||
maxSelections={imagePickerMax}
|
||||
onUseSelected={handleLibraryImageSelect}
|
||||
forceFilter="image"
|
||||
hideFilter
|
||||
/>
|
||||
<Lightbox
|
||||
isOpen={lightbox.lightboxOpen}
|
||||
onClose={lightbox.closeLightbox}
|
||||
mediaToken={lightbox.lightboxItem?.id}
|
||||
cdnUrl={lightbox.lightboxItem?.fullImage}
|
||||
mediaClass={lightbox.lightboxItem?.mediaClass}
|
||||
batchImageToken={lightbox.lightboxItem?.batchImageToken}
|
||||
showBatchCarousel={false}
|
||||
onNavigatePrev={lightbox.navigatePrev}
|
||||
onNavigateNext={lightbox.navigateNext}
|
||||
onDeleted={gallery.removeItem}
|
||||
/>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import { JobsApi, MediaFilesApi, OmniGenApi } from "@storyteller/api";
|
||||
import type { OmniGenSplatRequest } from "@storyteller/api";
|
||||
import type { GeneratedAsset } from "./create-world-store";
|
||||
|
||||
// ── Request params ───────────────────────────────────────────────────────
|
||||
|
||||
export interface GenerateSplatParams {
|
||||
model: string;
|
||||
prompt?: string;
|
||||
referenceImageMediaTokens?: string[];
|
||||
referenceVideoMediaToken?: string;
|
||||
isPanoramic?: boolean;
|
||||
disableRecaption?: boolean;
|
||||
}
|
||||
|
||||
// ── Enqueue generation ───────────────────────────────────────────────────
|
||||
|
||||
export async function enqueueSplatGeneration(
|
||||
params: GenerateSplatParams,
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
jobToken?: string;
|
||||
error?: string;
|
||||
errorCode?: number;
|
||||
}> {
|
||||
const body: OmniGenSplatRequest = {
|
||||
model: params.model,
|
||||
idempotency_token: crypto.randomUUID(),
|
||||
prompt: params.prompt ?? null,
|
||||
reference_image_media_tokens: params.referenceImageMediaTokens?.length
|
||||
? params.referenceImageMediaTokens
|
||||
: null,
|
||||
reference_video_media_token: params.referenceVideoMediaToken ?? null,
|
||||
is_panoramic: params.isPanoramic ?? null,
|
||||
disable_recaption: params.disableRecaption ?? null,
|
||||
};
|
||||
|
||||
try {
|
||||
const api = new OmniGenApi();
|
||||
const response = await api.generateSplat(body);
|
||||
if (response.success && response.inference_job_token) {
|
||||
return { success: true, jobToken: response.inference_job_token };
|
||||
}
|
||||
return { success: false, error: "Generation failed" };
|
||||
} catch (err: any) {
|
||||
return {
|
||||
success: false,
|
||||
error: err.message ?? "Request failed",
|
||||
errorCode: parseHttpStatusCode(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function parseHttpStatusCode(err: unknown): number | undefined {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const match = /status:\s*(\d+)/.exec(message);
|
||||
return match ? Number(match[1]) : undefined;
|
||||
}
|
||||
|
||||
// ── Poll for completion ──────────────────────────────────────────────────
|
||||
|
||||
async function pollJobResult(
|
||||
jobToken: string,
|
||||
): Promise<{ status: "pending" | "complete" | "failed"; assets: GeneratedAsset[]; error?: string }> {
|
||||
const jobsApi = new JobsApi();
|
||||
const response = await jobsApi.GetJobByToken({ token: jobToken });
|
||||
|
||||
if (!response.success || !response.data) {
|
||||
return { status: "pending", assets: [] };
|
||||
}
|
||||
|
||||
const state = response.data;
|
||||
const statusStr = state.status?.status?.toLowerCase() ?? "";
|
||||
|
||||
if (statusStr === "complete_success" || statusStr === "complete") {
|
||||
const result = state.maybe_result as Record<string, unknown> | undefined;
|
||||
const mediaLinks = (result as any)?.media_links;
|
||||
const entityToken = (result as any)?.entity_token as string | undefined;
|
||||
|
||||
if (mediaLinks?.cdn_url) {
|
||||
return {
|
||||
status: "complete",
|
||||
assets: [
|
||||
{
|
||||
media_token: entityToken ?? jobToken,
|
||||
cdn_url: mediaLinks.cdn_url,
|
||||
maybe_thumbnail_template: mediaLinks.maybe_thumbnail_template,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
if (entityToken) {
|
||||
try {
|
||||
const mediaApi = new MediaFilesApi();
|
||||
const mediaFile = await mediaApi.GetMediaFileByToken({
|
||||
mediaFileToken: entityToken,
|
||||
});
|
||||
const cdnUrl = (mediaFile.data as any)?.media_links?.cdn_url;
|
||||
if (cdnUrl) {
|
||||
return {
|
||||
status: "complete",
|
||||
assets: [{ media_token: entityToken, cdn_url: cdnUrl }],
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
|
||||
return { status: "complete", assets: [] };
|
||||
}
|
||||
|
||||
if (
|
||||
statusStr.includes("fail") ||
|
||||
statusStr.includes("error") ||
|
||||
statusStr === "dead"
|
||||
) {
|
||||
return {
|
||||
status: "failed",
|
||||
assets: [],
|
||||
error:
|
||||
state.status?.maybe_failure_message ??
|
||||
state.status?.maybe_extra_status_description ??
|
||||
"Generation failed",
|
||||
};
|
||||
}
|
||||
|
||||
return { status: "pending", assets: [] };
|
||||
}
|
||||
|
||||
// ── Polling controller ───────────────────────────────────────────────────
|
||||
|
||||
// Splat / world generation is slow, so allow a generous window.
|
||||
export function startPolling(
|
||||
jobToken: string,
|
||||
onComplete: (assets: GeneratedAsset[]) => void,
|
||||
onError: (reason: string) => void,
|
||||
intervalMs = 5000,
|
||||
maxAttempts = 180,
|
||||
): () => void {
|
||||
let attempts = 0;
|
||||
let stopped = false;
|
||||
|
||||
const poll = async () => {
|
||||
if (stopped) return;
|
||||
attempts++;
|
||||
|
||||
try {
|
||||
const result = await pollJobResult(jobToken);
|
||||
if (stopped) return;
|
||||
|
||||
if (result.status === "complete") {
|
||||
onComplete(result.assets);
|
||||
return;
|
||||
}
|
||||
if (result.status === "failed") {
|
||||
onError(result.error ?? "Generation failed");
|
||||
return;
|
||||
}
|
||||
if (attempts >= maxAttempts) {
|
||||
onError("Generation timed out");
|
||||
return;
|
||||
}
|
||||
|
||||
setTimeout(poll, intervalMs);
|
||||
} catch {
|
||||
if (!stopped && attempts < maxAttempts) {
|
||||
setTimeout(poll, intervalMs * 2);
|
||||
} else if (!stopped) {
|
||||
onError("Network error during polling");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
setTimeout(poll, 2000);
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { default } from "./create-world";
|
||||
@@ -304,6 +304,137 @@ export interface OmniGenProviderEntry {
|
||||
models: OmniGenProviderModelEntry[];
|
||||
}
|
||||
|
||||
// ── Mesh (3D object) request / response types ────────────────────────────
|
||||
|
||||
// Shared body for POST /v1/omni_gen/cost/mesh and /v1/omni_gen/generate/mesh.
|
||||
export interface OmniGenMeshRequest {
|
||||
// Required despite being nullable (backend rejects when absent).
|
||||
model: string;
|
||||
idempotency_token?: string | null;
|
||||
prompt?: string | null;
|
||||
// Primary/front input image, or the sketch image for sketch-to-3D.
|
||||
reference_image_media_tokens?: string[] | null;
|
||||
// Multi-view input (one image per side).
|
||||
front_image_media_token?: string | null;
|
||||
back_image_media_token?: string | null;
|
||||
left_image_media_token?: string | null;
|
||||
right_image_media_token?: string | null;
|
||||
// Input mesh file for mesh-to-mesh models (part splitting, retopology).
|
||||
input_mesh_media_token?: string | null;
|
||||
// "normal" | "low_poly" | "geometry"
|
||||
mesh_output_type?: string | null;
|
||||
// "triangle" | "quad"
|
||||
polygon_type?: string | null;
|
||||
face_count?: number | null;
|
||||
enable_pbr?: boolean | null;
|
||||
enable_texture?: boolean | null;
|
||||
// "standard" | "detailed"
|
||||
texture_quality?: string | null;
|
||||
geometry_quality?: string | null;
|
||||
}
|
||||
|
||||
export interface OmniGenMeshCostResponse {
|
||||
success: boolean;
|
||||
cost_in_credits?: number | null;
|
||||
cost_in_usd_cents?: number | null;
|
||||
has_watermark: boolean;
|
||||
is_free: boolean;
|
||||
is_rate_limited: boolean;
|
||||
is_unlimited: boolean;
|
||||
failures_are_refunded?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OmniGenMeshGenerateResponse {
|
||||
success: boolean;
|
||||
inference_job_token: string;
|
||||
all_job_tokens: string[];
|
||||
}
|
||||
|
||||
export interface OmniGenMeshModelInfo {
|
||||
model: string;
|
||||
// ModelCreator enum as a snake_case string, e.g. "tencent".
|
||||
model_creator?: string | null;
|
||||
full_name?: string | null;
|
||||
extra_info_short?: string | null;
|
||||
extra_info?: string | null;
|
||||
text_prompt_supported?: boolean | null;
|
||||
image_input_supported?: boolean | null;
|
||||
sketch_input_supported?: boolean | null;
|
||||
multi_view_supported?: boolean | null;
|
||||
mesh_input_supported?: boolean | null;
|
||||
// CommonMeshOutputType values: "normal" | "low_poly" | "geometry".
|
||||
mesh_output_types?: string[] | null;
|
||||
// CommonPolygonType values: "triangle" | "quad".
|
||||
polygon_types?: string[] | null;
|
||||
face_count_supported?: boolean | null;
|
||||
pbr_supported?: boolean | null;
|
||||
texture_toggle_supported?: boolean | null;
|
||||
texture_quality_supported?: boolean | null;
|
||||
geometry_quality_supported?: boolean | null;
|
||||
is_disabled?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OmniGenMeshModelsResponse {
|
||||
success: boolean;
|
||||
models: OmniGenMeshModelInfo[];
|
||||
providers: OmniGenProviderEntry[];
|
||||
}
|
||||
|
||||
// ── Splat (3D world) request / response types ────────────────────────────
|
||||
|
||||
// Shared body for POST /v1/omni_gen/cost/splat and /v1/omni_gen/generate/splat.
|
||||
export interface OmniGenSplatRequest {
|
||||
// Required despite being nullable (backend rejects when absent).
|
||||
model: string;
|
||||
idempotency_token?: string | null;
|
||||
prompt?: string | null;
|
||||
// Multi-view input of the same world.
|
||||
reference_image_media_tokens?: string[] | null;
|
||||
reference_video_media_token?: string | null;
|
||||
// Whether the single reference image is a 360-degree panorama.
|
||||
is_panoramic?: boolean | null;
|
||||
disable_recaption?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OmniGenSplatCostResponse {
|
||||
success: boolean;
|
||||
cost_in_credits?: number | null;
|
||||
cost_in_usd_cents?: number | null;
|
||||
has_watermark: boolean;
|
||||
is_free: boolean;
|
||||
is_rate_limited: boolean;
|
||||
is_unlimited: boolean;
|
||||
failures_are_refunded?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OmniGenSplatGenerateResponse {
|
||||
success: boolean;
|
||||
inference_job_token: string;
|
||||
all_job_tokens: string[];
|
||||
}
|
||||
|
||||
export interface OmniGenSplatModelInfo {
|
||||
model: string;
|
||||
// ModelCreator enum as a snake_case string, e.g. "world_labs".
|
||||
model_creator?: string | null;
|
||||
full_name?: string | null;
|
||||
extra_info_short?: string | null;
|
||||
extra_info?: string | null;
|
||||
text_prompt_supported?: boolean | null;
|
||||
image_references_supported?: boolean | null;
|
||||
image_references_max?: number | null;
|
||||
video_reference_supported?: boolean | null;
|
||||
panorama_supported?: boolean | null;
|
||||
disable_recaption_supported?: boolean | null;
|
||||
is_disabled?: boolean | null;
|
||||
}
|
||||
|
||||
export interface OmniGenSplatModelsResponse {
|
||||
success: boolean;
|
||||
models: OmniGenSplatModelInfo[];
|
||||
providers: OmniGenProviderEntry[];
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** Strip keys whose value is null or undefined so the server only sees
|
||||
@@ -347,6 +478,26 @@ export class OmniGenApi extends ApiManager {
|
||||
});
|
||||
}
|
||||
|
||||
public async getMeshModels(
|
||||
provider?: string,
|
||||
): Promise<OmniGenMeshModelsResponse> {
|
||||
const query = provider ? { provider } : undefined;
|
||||
return this.get<OmniGenMeshModelsResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/models/mesh`,
|
||||
query,
|
||||
});
|
||||
}
|
||||
|
||||
public async getSplatModels(
|
||||
provider?: string,
|
||||
): Promise<OmniGenSplatModelsResponse> {
|
||||
const query = provider ? { provider } : undefined;
|
||||
return this.get<OmniGenSplatModelsResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/models/splat`,
|
||||
query,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Cost estimates ───────────────────────────────────────────────────
|
||||
|
||||
public async estimateImageCost(
|
||||
@@ -376,6 +527,24 @@ export class OmniGenApi extends ApiManager {
|
||||
});
|
||||
}
|
||||
|
||||
public async estimateMeshCost(
|
||||
body: OmniGenMeshRequest,
|
||||
): Promise<OmniGenMeshCostResponse> {
|
||||
return this.post<Record<string, unknown>, OmniGenMeshCostResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/cost/mesh`,
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
|
||||
public async estimateSplatCost(
|
||||
body: OmniGenSplatRequest,
|
||||
): Promise<OmniGenSplatCostResponse> {
|
||||
return this.post<Record<string, unknown>, OmniGenSplatCostResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/cost/splat`,
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Generation ───────────────────────────────────────────────────────
|
||||
|
||||
public async generateImage(
|
||||
@@ -404,4 +573,22 @@ export class OmniGenApi extends ApiManager {
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
|
||||
public async generateMesh(
|
||||
body: OmniGenMeshRequest,
|
||||
): Promise<OmniGenMeshGenerateResponse> {
|
||||
return this.post<Record<string, unknown>, OmniGenMeshGenerateResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/generate/mesh`,
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
|
||||
public async generateSplat(
|
||||
body: OmniGenSplatRequest,
|
||||
): Promise<OmniGenSplatGenerateResponse> {
|
||||
return this.post<Record<string, unknown>, OmniGenSplatGenerateResponse>({
|
||||
endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/generate/splat`,
|
||||
body: stripNulls(body),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,11 @@
|
||||
// Canonical data shapes for the merged generation feed (in-progress / failed /
|
||||
// completed). Hosts (webapp, desktop) map their own job sources into these.
|
||||
|
||||
export type GenerationMediaClass = "image" | "video" | "audio";
|
||||
export type GenerationMediaClass =
|
||||
| "image"
|
||||
| "video"
|
||||
| "audio"
|
||||
| "dimensional";
|
||||
|
||||
/** Plural noun for batch captions, e.g. "Generating 4 images". */
|
||||
export function batchNoun(mediaClass: GenerationMediaClass): string {
|
||||
@@ -10,6 +14,8 @@ export function batchNoun(mediaClass: GenerationMediaClass): string {
|
||||
return "videos";
|
||||
case "audio":
|
||||
return "audio clips";
|
||||
case "dimensional":
|
||||
return "3D models";
|
||||
default:
|
||||
return "images";
|
||||
}
|
||||
|
||||
@@ -27,12 +27,32 @@ const getLabel = (item: any) => {
|
||||
|
||||
// ── Hook ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// Pull a cover image / screenshot URL off a dimensional (3D) library item. The
|
||||
// item's own cdn_url is the .glb/.spz asset, so 3D cards render the separate
|
||||
// cover image the backend attaches instead. Mirrors the gallery-modal logic.
|
||||
function get3DCoverThumbnail(item: any): string | null {
|
||||
const cover = item?.cover_image;
|
||||
if (!cover) return null;
|
||||
return (
|
||||
cover.maybe_links?.maybe_thumbnail_template?.replace("{WIDTH}", "512") ??
|
||||
cover.maybe_links?.cdn_url ??
|
||||
cover.maybe_cover_image_public_bucket_url ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
export function useGalleryData(options: {
|
||||
username: string | null;
|
||||
filterMediaClasses: FilterMediaClasses[];
|
||||
excludeUploads?: boolean;
|
||||
// When set, dimensional (3D) items are kept only if their model belongs to
|
||||
// this list. The library API filters by media class ("dimensional") but can't
|
||||
// tell mesh objects from splat worlds apart, so the two 3D create pages pass
|
||||
// their own model ids to keep each feed showing only its own product.
|
||||
filterModelIds?: string[];
|
||||
}) {
|
||||
const { username, filterMediaClasses, excludeUploads } = options;
|
||||
const { username, filterMediaClasses, excludeUploads, filterModelIds } =
|
||||
options;
|
||||
|
||||
const [items, setItems] = useState<GalleryItem[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
@@ -44,15 +64,16 @@ export function useGalleryData(options: {
|
||||
const api = useMemo(() => new GalleryModalApi(), []);
|
||||
|
||||
const mapApiItem = useCallback((item: any): GalleryItem => {
|
||||
// 3D and audio media have no image thumbnail — their cards render a
|
||||
// placeholder icon / waveform player instead.
|
||||
const hasImageThumbnail =
|
||||
item.media_class !== "dimensional" && item.media_class !== "audio";
|
||||
const thumbnail = hasImageThumbnail
|
||||
? getMediaThumbnail(item.media_links, item.media_class, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
})
|
||||
: null;
|
||||
// 3D uses the cover image / screenshot; audio has no thumbnail (its card
|
||||
// renders a waveform player); everything else uses the media thumbnail.
|
||||
const thumbnail =
|
||||
item.media_class === "audio"
|
||||
? null
|
||||
: item.media_class === "dimensional"
|
||||
? get3DCoverThumbnail(item)
|
||||
: getMediaThumbnail(item.media_links, item.media_class, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
|
||||
return {
|
||||
id: item.token,
|
||||
@@ -85,11 +106,25 @@ export function useGalleryData(options: {
|
||||
});
|
||||
|
||||
if (response.success && response.data) {
|
||||
const modelIdSet =
|
||||
filterModelIds && filterModelIds.length
|
||||
? new Set(filterModelIds)
|
||||
: null;
|
||||
|
||||
const newItems = response.data
|
||||
.filter(
|
||||
(item: any) =>
|
||||
item.media_type !== FilterMediaType.SCENE_JSON &&
|
||||
!(excludeUploads && item.origin_category === "upload"),
|
||||
!(excludeUploads && item.origin_category === "upload") &&
|
||||
// Split 3D history by model so the object and world pages don't
|
||||
// show each other's generations. Non-dimensional items and items
|
||||
// without a known model are left untouched.
|
||||
!(
|
||||
modelIdSet &&
|
||||
item.media_class === "dimensional" &&
|
||||
item.maybe_model_type &&
|
||||
!modelIdSet.has(item.maybe_model_type)
|
||||
),
|
||||
)
|
||||
.map(mapApiItem);
|
||||
|
||||
@@ -112,7 +147,15 @@ export function useGalleryData(options: {
|
||||
setIsInitialLoading(false);
|
||||
isLoadingRef.current = false;
|
||||
},
|
||||
[username, filterMediaClasses, pageIndex, api, mapApiItem, excludeUploads],
|
||||
[
|
||||
username,
|
||||
filterMediaClasses,
|
||||
pageIndex,
|
||||
api,
|
||||
mapApiItem,
|
||||
excludeUploads,
|
||||
filterModelIds,
|
||||
],
|
||||
);
|
||||
|
||||
// Initial load + filter change. When logged out (no username), clear the
|
||||
@@ -129,7 +172,8 @@ export function useGalleryData(options: {
|
||||
setHasMore(true);
|
||||
setIsInitialLoading(true);
|
||||
loadItems(true);
|
||||
}, [username, JSON.stringify(filterMediaClasses)]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [username, JSON.stringify(filterMediaClasses), JSON.stringify(filterModelIds)]);
|
||||
|
||||
const loadMore = useCallback(() => {
|
||||
if (hasMore && !isLoadingRef.current) {
|
||||
|
||||
@@ -50,6 +50,7 @@ const DEFAULT_DURATION_MS: Record<GenerationMediaClass, number> = {
|
||||
image: 30000,
|
||||
video: 900000,
|
||||
audio: 120000,
|
||||
dimensional: 120000,
|
||||
};
|
||||
|
||||
// Cache per-task durations
|
||||
@@ -57,11 +58,24 @@ const taskDurationCache = new Map<string, number>();
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
// The backend `inference_category` strings for 3D are "object_generation"
|
||||
// (mesh) and "splat_generation" (gaussian splats). Both render as the
|
||||
// "dimensional" media class, but stay distinct here so the two 3D create pages
|
||||
// can each filter to only their own generations.
|
||||
export type JobMediaFilter =
|
||||
| "image"
|
||||
| "video"
|
||||
| "audio"
|
||||
| "object"
|
||||
| "splat";
|
||||
|
||||
function getJobMediaType(
|
||||
job: Job,
|
||||
): "image" | "video" | "audio" | "other" {
|
||||
): JobMediaFilter | "other" {
|
||||
const cat = job.request.inference_category?.toLowerCase() ?? "";
|
||||
if (cat.includes("video")) return "video";
|
||||
if (cat.includes("object")) return "object";
|
||||
if (cat.includes("splat") || cat.includes("gaussian")) return "splat";
|
||||
if (cat.includes("image")) return "image";
|
||||
if (cat.includes("audio")) return "audio";
|
||||
return "other";
|
||||
@@ -69,9 +83,25 @@ function getJobMediaType(
|
||||
|
||||
function getJobMediaClass(job: Job): GenerationMediaClass {
|
||||
const inferred = getJobMediaType(job);
|
||||
if (inferred === "object" || inferred === "splat") return "dimensional";
|
||||
return inferred === "other" ? "image" : inferred;
|
||||
}
|
||||
|
||||
// Pull a cover image / screenshot URL off a 3D result (job result or a batch
|
||||
// media file). The raw cdn_url is the .glb/.spz asset itself, so 3D cards use
|
||||
// the separate cover image the backend attaches; falls back to null (the card
|
||||
// then shows its 3D placeholder).
|
||||
function get3DCoverThumbnail(source: any): string | null {
|
||||
const cover = source?.cover_image;
|
||||
if (!cover) return null;
|
||||
return (
|
||||
cover.maybe_links?.maybe_thumbnail_template?.replace("{WIDTH}", "512") ??
|
||||
cover.maybe_links?.cdn_url ??
|
||||
cover.maybe_cover_image_public_bucket_url ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function getModelLabel(job: Job, promptsMap?: Map<string, Prompts>): string {
|
||||
const promptToken = job.request.maybe_prompt_token;
|
||||
const cachedPrompt = promptToken
|
||||
@@ -204,13 +234,16 @@ function jobToGalleryItem(
|
||||
if (!result?.entity_token) return null;
|
||||
|
||||
const mediaClass = getJobMediaClass(job);
|
||||
// Audio has no image thumbnail — its card renders a waveform player.
|
||||
// Audio has no image thumbnail — its card renders a waveform player. 3D uses
|
||||
// the result's cover image / screenshot (the cdn_url is the mesh/splat file).
|
||||
const thumbnail =
|
||||
mediaClass === "audio"
|
||||
? null
|
||||
: getMediaThumbnail(result.media_links, mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
: mediaClass === "dimensional"
|
||||
? get3DCoverThumbnail(result)
|
||||
: getMediaThumbnail(result.media_links, mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
});
|
||||
|
||||
return {
|
||||
id: result.entity_token,
|
||||
@@ -249,9 +282,11 @@ async function expandBatchItems(
|
||||
const thumbnail =
|
||||
item.mediaClass === "audio"
|
||||
? null
|
||||
: getMediaThumbnail(file.media_links, item.mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
}) || cdnUrl;
|
||||
: item.mediaClass === "dimensional"
|
||||
? get3DCoverThumbnail(file)
|
||||
: getMediaThumbnail(file.media_links, item.mediaClass, {
|
||||
size: THUMBNAIL_SIZES.LARGE,
|
||||
}) || cdnUrl;
|
||||
return {
|
||||
id: file.token,
|
||||
label: item.label,
|
||||
@@ -273,7 +308,7 @@ async function expandBatchItems(
|
||||
// ── Hook ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export function useGenerationJobs(options: {
|
||||
mediaType: GenerationMediaClass;
|
||||
mediaType: JobMediaFilter;
|
||||
enabled?: boolean;
|
||||
}) {
|
||||
const { mediaType, enabled = true } = options;
|
||||
|
||||
@@ -4,6 +4,8 @@ import type {
|
||||
OmniGenAudioModelDetails,
|
||||
OmniGenImageModelInfo,
|
||||
OmniGenVideoModelInfo,
|
||||
OmniGenMeshModelInfo,
|
||||
OmniGenSplatModelInfo,
|
||||
} from "@storyteller/api";
|
||||
|
||||
// Shown when the OmniGen service is unavailable (e.g. returns 5xx). Separate
|
||||
@@ -44,6 +46,16 @@ let audioModelsFetching = false;
|
||||
let audioModelsError: string | null = null;
|
||||
let audioModelsListeners: Array<() => void> = [];
|
||||
|
||||
let meshModelsCache: OmniGenMeshModelInfo[] | null = null;
|
||||
let meshModelsFetching = false;
|
||||
let meshModelsError: string | null = null;
|
||||
let meshModelsListeners: Array<() => void> = [];
|
||||
|
||||
let splatModelsCache: OmniGenSplatModelInfo[] | null = null;
|
||||
let splatModelsFetching = false;
|
||||
let splatModelsError: string | null = null;
|
||||
let splatModelsListeners: Array<() => void> = [];
|
||||
|
||||
function notifyImageListeners() {
|
||||
imageModelsListeners.forEach((cb) => cb());
|
||||
imageModelsListeners = [];
|
||||
@@ -59,6 +71,16 @@ function notifyAudioListeners() {
|
||||
audioModelsListeners = [];
|
||||
}
|
||||
|
||||
function notifyMeshListeners() {
|
||||
meshModelsListeners.forEach((cb) => cb());
|
||||
meshModelsListeners = [];
|
||||
}
|
||||
|
||||
function notifySplatListeners() {
|
||||
splatModelsListeners.forEach((cb) => cb());
|
||||
splatModelsListeners = [];
|
||||
}
|
||||
|
||||
function fetchImageModelsOnce() {
|
||||
if (imageModelsCache || imageModelsFetching) return;
|
||||
imageModelsFetching = true;
|
||||
@@ -140,6 +162,60 @@ function fetchAudioModelsOnce() {
|
||||
);
|
||||
}
|
||||
|
||||
function fetchMeshModelsOnce() {
|
||||
if (meshModelsCache || meshModelsFetching) return;
|
||||
meshModelsFetching = true;
|
||||
meshModelsError = null;
|
||||
const api = new OmniGenApi();
|
||||
api.getMeshModels().then(
|
||||
(res) => {
|
||||
meshModelsFetching = false;
|
||||
if (res.success && res.models?.length) {
|
||||
meshModelsCache = res.models.filter((m) => m.is_disabled !== true);
|
||||
} else {
|
||||
meshModelsError = "No mesh models returned from API";
|
||||
console.warn("[OmniGen] Mesh models response:", res);
|
||||
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
|
||||
}
|
||||
notifyMeshListeners();
|
||||
},
|
||||
(err) => {
|
||||
meshModelsFetching = false;
|
||||
meshModelsError = err?.message ?? "Failed to fetch mesh models";
|
||||
console.error("[OmniGen] Failed to fetch mesh models:", err);
|
||||
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
|
||||
notifyMeshListeners();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function fetchSplatModelsOnce() {
|
||||
if (splatModelsCache || splatModelsFetching) return;
|
||||
splatModelsFetching = true;
|
||||
splatModelsError = null;
|
||||
const api = new OmniGenApi();
|
||||
api.getSplatModels().then(
|
||||
(res) => {
|
||||
splatModelsFetching = false;
|
||||
if (res.success && res.models?.length) {
|
||||
splatModelsCache = res.models.filter((m) => m.is_disabled !== true);
|
||||
} else {
|
||||
splatModelsError = "No splat models returned from API";
|
||||
console.warn("[OmniGen] Splat models response:", res);
|
||||
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
|
||||
}
|
||||
notifySplatListeners();
|
||||
},
|
||||
(err) => {
|
||||
splatModelsFetching = false;
|
||||
splatModelsError = err?.message ?? "Failed to fetch splat models";
|
||||
console.error("[OmniGen] Failed to fetch splat models:", err);
|
||||
errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE);
|
||||
notifySplatListeners();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// ── Hooks ────────────────────────────────────────────────────────────────
|
||||
|
||||
export function useOmniGenImageModels(): {
|
||||
@@ -267,3 +343,85 @@ export function useOmniGenAudioModels(): {
|
||||
|
||||
return { models, isLoading, error };
|
||||
}
|
||||
|
||||
export function useOmniGenMeshModels(): {
|
||||
models: OmniGenMeshModelInfo[];
|
||||
isLoading: boolean;
|
||||
error: string | null;
|
||||
} {
|
||||
const [models, setModels] = useState<OmniGenMeshModelInfo[]>(
|
||||
meshModelsCache ?? [],
|
||||
);
|
||||
const [isLoading, setIsLoading] = useState(!meshModelsCache);
|
||||
const [error, setError] = useState<string | null>(meshModelsError);
|
||||
|
||||
useEffect(() => {
|
||||
if (meshModelsCache) {
|
||||
setModels(meshModelsCache);
|
||||
setIsLoading(false);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const onReady = () => {
|
||||
setIsLoading(false);
|
||||
if (meshModelsCache) {
|
||||
setModels(meshModelsCache);
|
||||
setError(null);
|
||||
} else {
|
||||
setError(meshModelsError);
|
||||
}
|
||||
};
|
||||
|
||||
meshModelsListeners.push(onReady);
|
||||
fetchMeshModelsOnce();
|
||||
|
||||
return () => {
|
||||
meshModelsListeners = meshModelsListeners.filter((cb) => cb !== onReady);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { models, isLoading, error };
|
||||
}
|
||||
|
||||
export function useOmniGenSplatModels(): {
|
||||
models: OmniGenSplatModelInfo[];
|
||||
isLoading: boolean;
|
||||
error: string | null;
|
||||
} {
|
||||
const [models, setModels] = useState<OmniGenSplatModelInfo[]>(
|
||||
splatModelsCache ?? [],
|
||||
);
|
||||
const [isLoading, setIsLoading] = useState(!splatModelsCache);
|
||||
const [error, setError] = useState<string | null>(splatModelsError);
|
||||
|
||||
useEffect(() => {
|
||||
if (splatModelsCache) {
|
||||
setModels(splatModelsCache);
|
||||
setIsLoading(false);
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const onReady = () => {
|
||||
setIsLoading(false);
|
||||
if (splatModelsCache) {
|
||||
setModels(splatModelsCache);
|
||||
setError(null);
|
||||
} else {
|
||||
setError(splatModelsError);
|
||||
}
|
||||
};
|
||||
|
||||
splatModelsListeners.push(onReady);
|
||||
fetchSplatModelsOnce();
|
||||
|
||||
return () => {
|
||||
splatModelsListeners = splatModelsListeners.filter(
|
||||
(cb) => cb !== onReady,
|
||||
);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { models, isLoading, error };
|
||||
}
|
||||
Reference in new issue
Block a user