From d538bc2a68b702e02aa31202ae39ae2d9d0d8102 Mon Sep 17 00:00:00 2001 From: BFlat Date: Wed, 8 Jul 2026 23:03:11 +0700 Subject: [PATCH] Add 3D Object & 3D World generation to the webapp (#1756) Adds two new create pages for 3D generation, 3D Object (meshes) and 3D World (gaussian splats), wired up to the new omni-gen mesh/splat endpoints, plus the navigation, icons, and polish to go with them. ### New create pages - 3D Object (`/create-object`): turn a prompt or image into a 3D model. - 3D World (`/create-world`): turn a prompt or image into an explorable world. - Both mirror the existing Create Image/Video experience: prompt box with model selector and per-model options in the bottom bar, and a live feed of in-progress, completed, and failed generations. - Options adapt to the selected model, e.g. output type, polygon type, geometry/texture quality, face count, PBR and texture toggles, multi-view reference angles (front/back/left/right), mesh-file input, reference image/video, panorama, and prompt-only where supported. - Multi-view and reference slots support the same Upload / Pick-from-library flow (desktop hover and mobile) as the image rows. - Completed cards show the model's cover screenshot; clicking one opens it in the interactive 3D viewer. ### Navigation and discovery - Sidebar reorganized into two groups: Create (Image, Video, Audio, 3D Object, 3D World) and a new Studio group (Edit Image, Edit 3D, Edit Video, BG Change, Moodboard). - Added 3D Object and 3D World cards to the Home page. - Added breadcrumbs for the new pages. - Added a My Library button to the top bar, next to the credits and task-queue chips. ### Model icons - Restored the Hunyuan (Tencent) and World Labs provider icons. - Added Tripo, Meshy, and Rodin (Hyper3D) icons. - Applied to both the webapp and desktop app. ### Fixes and polish - Fixed the 3D-model icons showing a generic placeholder (missing cyan color plus creator mappings). - Fixed the Upload / Pick-from-library hover popover being clipped inside glass panels. - Reference-image rows in the 3D pages now align their fields to the right with the title on the left, matching the other create pages. - Prevented 3D-model cover screenshots from showing up as their own "3D object" cards across the 3D feeds, the Library page, and the gallery picker. --- .../public/images/services/hyper3d.svg | 1 + .../public/images/services/meshy.svg | 1 + .../public/images/services/tripo.svg | 1 + frontend/apps/artcraft-webapp/src/app/app.tsx | 4 + .../GenerationGalleryGrid.tsx | 6 +- .../GenerationGalleryList.tsx | 6 +- .../components/prompt-box/ImagePromptRow.tsx | 2 +- .../src/components/prompt-box/PromptBox.tsx | 9 +- .../src/components/prompt-box/index.ts | 2 +- .../src/components/sidebar/app-sidebar.tsx | 27 +- .../src/components/topbar/TopBarActions.tsx | 11 +- .../src/components/topbar/breadcrumbs.tsx | 2 + .../src/components/ui/sidebar.tsx | 6 +- .../src/lib/cost-estimate-api.ts | 142 +++- .../components/MeshInputsRow.tsx | 314 ++++++++ .../components/MeshOptionPickers.tsx | 175 +++++ .../create-object/create-object-store.ts | 169 +++++ .../src/pages/create-object/create-object.tsx | 708 ++++++++++++++++++ .../create-object/generate-object-api.ts | 196 +++++ .../src/pages/create-object/index.ts | 1 + .../components/ReferenceVideoSlot.tsx | 108 +++ .../pages/create-world/create-world-store.ts | 146 ++++ .../src/pages/create-world/create-world.tsx | 581 ++++++++++++++ .../pages/create-world/generate-world-api.ts | 181 +++++ .../src/pages/create-world/index.ts | 1 + .../artcraft-webapp/src/pages/home/home.tsx | 19 + .../src/pages/library/library.tsx | 11 +- .../apps/artcraft-webapp/tailwind.config.js | 1 + .../resources/images/services/hyper3d.svg | 1 + .../resources/images/services/meshy.svg | 1 + .../resources/images/services/tripo.svg | 1 + frontend/libs/api/src/lib/OmniGenApi.ts | 187 +++++ .../common/src/lib/utils/thumbnail-utils.ts | 13 + .../gallery-modal/src/lib/gallery-modal.tsx | 10 +- .../generation-list/src/lib/types.ts | 8 +- .../generation-list/src/lib/useGalleryData.ts | 70 +- .../src/lib/useGenerationJobs.ts | 53 +- .../components/tooltip/src/lib/tooltip.tsx | 8 +- .../classes/metadata/ModelCreatorIconForId.ts | 19 +- .../libs/omni-gen/src/lib/omni-gen-hooks.ts | 158 ++++ 40 files changed, 3309 insertions(+), 51 deletions(-) create mode 100644 frontend/apps/artcraft-webapp/public/images/services/hyper3d.svg create mode 100644 frontend/apps/artcraft-webapp/public/images/services/meshy.svg create mode 100644 frontend/apps/artcraft-webapp/public/images/services/tripo.svg create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-object/components/MeshInputsRow.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-object/components/MeshOptionPickers.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-object/create-object-store.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-object/create-object.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-object/generate-object-api.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-object/index.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-world/components/ReferenceVideoSlot.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-world/create-world-store.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-world/create-world.tsx create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-world/generate-world-api.ts create mode 100644 frontend/apps/artcraft-webapp/src/pages/create-world/index.ts create mode 100644 frontend/apps/artcraft/app/public/resources/images/services/hyper3d.svg create mode 100644 frontend/apps/artcraft/app/public/resources/images/services/meshy.svg create mode 100644 frontend/apps/artcraft/app/public/resources/images/services/tripo.svg diff --git a/frontend/apps/artcraft-webapp/public/images/services/hyper3d.svg b/frontend/apps/artcraft-webapp/public/images/services/hyper3d.svg new file mode 100644 index 0000000000..aa99495e5f --- /dev/null +++ b/frontend/apps/artcraft-webapp/public/images/services/hyper3d.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/apps/artcraft-webapp/public/images/services/meshy.svg b/frontend/apps/artcraft-webapp/public/images/services/meshy.svg new file mode 100644 index 0000000000..da77f22eb2 --- /dev/null +++ b/frontend/apps/artcraft-webapp/public/images/services/meshy.svg @@ -0,0 +1 @@ +Meshy \ No newline at end of file diff --git a/frontend/apps/artcraft-webapp/public/images/services/tripo.svg b/frontend/apps/artcraft-webapp/public/images/services/tripo.svg new file mode 100644 index 0000000000..40f4c37190 --- /dev/null +++ b/frontend/apps/artcraft-webapp/public/images/services/tripo.svg @@ -0,0 +1 @@ +Tripo \ No newline at end of file diff --git a/frontend/apps/artcraft-webapp/src/app/app.tsx b/frontend/apps/artcraft-webapp/src/app/app.tsx index a8cd520333..10d18c1d13 100644 --- a/frontend/apps/artcraft-webapp/src/app/app.tsx +++ b/frontend/apps/artcraft-webapp/src/app/app.tsx @@ -15,6 +15,8 @@ import { ToastContainer } from "../components/toast/toast"; import CreateImage from "../pages/create-image"; import CreateVideo from "../pages/create-video"; import CreateAudio from "../pages/create-audio"; +import CreateObject from "../pages/create-object"; +import CreateWorld from "../pages/create-world"; import CreateVFX from "../pages/create-vfx"; import PageScene from "../pages/pagescene"; import PageDraw from "../pages/pagedraw"; @@ -155,6 +157,8 @@ export function App() { } /> } /> } /> + } /> + } /> } /> } /> } /> diff --git a/frontend/apps/artcraft-webapp/src/components/generation-gallery/GenerationGalleryGrid.tsx b/frontend/apps/artcraft-webapp/src/components/generation-gallery/GenerationGalleryGrid.tsx index 6ad03427af..73647cbb05 100644 --- a/frontend/apps/artcraft-webapp/src/components/generation-gallery/GenerationGalleryGrid.tsx +++ b/frontend/apps/artcraft-webapp/src/components/generation-gallery/GenerationGalleryGrid.tsx @@ -65,11 +65,15 @@ function CardRecreateButton({ mediaClass, kind, }: RecreateSlotContext) { + // Recreate only targets the image/video create pages; 3D generations have no + // recreate flow, so map to a valid class for the hook and render nothing. const { isRecreating, handleRecreate } = useRecreateFromPromptToken( promptToken, - mediaClass, + mediaClass === "dimensional" ? "image" : mediaClass, ); + if (mediaClass === "dimensional") return null; + if (kind === "failed") { return ( + + ) : uploading ? ( +
+ +
+ ) : ( + inputRef.current?.click()} + onPickFromLibrary={onPickFromLibrary} + /> + )} + + ); +} + +function MeshFileSlot({ + mesh, + onChange, +}: { + mesh?: RefImage; + onChange: (img?: RefImage) => void; +}) { + const inputRef = useRef(null); + const [uploading, setUploading] = useState(false); + + const handleUpload = async (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (!file) return; + setUploading(true); + try { + const api = new MediaUploadApi(); + const response = await api.UploadNewEngineAsset({ + file, + fileName: file.name || `input-mesh-${Date.now()}`, + uuid: crypto.randomUUID(), + engine_category: FilterEngineCategories.OBJECT, + maybe_title: "input_mesh", + }); + if (response?.success && response.data) { + onChange({ + id: Math.random().toString(36).substring(7), + url: "", + file, + mediaToken: response.data, + }); + } + } catch { + // ignore — user can retry + } finally { + setUploading(false); + if (inputRef.current) inputRef.current.value = ""; + } + }; + + return ( + + + {mesh ? ( +
+ + +
+ ) : uploading ? ( +
+ +
+ ) : ( + + )} + + ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/create-object/components/MeshOptionPickers.tsx b/frontend/apps/artcraft-webapp/src/pages/create-object/components/MeshOptionPickers.tsx new file mode 100644 index 0000000000..7c091040ae --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-object/components/MeshOptionPickers.tsx @@ -0,0 +1,175 @@ +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { + faCube, + faObjectGroup, + faWandMagicSparkles, + faImage, + faGem, +} from "@fortawesome/pro-solid-svg-icons"; +import type { IconDefinition } from "@fortawesome/fontawesome-svg-core"; +import { PopoverItem, PopoverMenu } from "@storyteller/ui-popover"; +import { Tooltip } from "@storyteller/ui-tooltip"; + +// ── Enum label maps (backend value → human label) ────────────────────────── + +const MESH_OUTPUT_TYPE_LABELS: Record = { + normal: "Standard", + low_poly: "Low Poly", + geometry: "Geometry only", +}; + +const POLYGON_TYPE_LABELS: Record = { + triangle: "Triangle", + quad: "Quad", +}; + +const MESH_QUALITY_LABELS: Record = { + standard: "Standard", + detailed: "Detailed", +}; + +// Preset target face counts (poly budget). Backend accepts any u64; these are +// sensible presets exposed in the UI. +const FACE_COUNT_PRESETS: number[] = [10000, 30000, 50000, 100000, 200000]; + +// ── Generic enum picker ──────────────────────────────────────────────────── + +interface EnumPickerProps { + options: string[]; + current?: string; + onSelect: (value: string) => void; + labels: Record; + tooltip: string; + panelTitle: string; + icon: IconDefinition; +} + +function EnumPicker({ + options, + current, + onSelect, + labels, + tooltip, + panelTitle, + icon, +}: EnumPickerProps) { + const labelToValue: Record = Object.fromEntries( + options.map((v) => [labels[v] ?? v, v]), + ); + + const items: PopoverItem[] = options.map((v) => ({ + label: labels[v] ?? v, + selected: current === v, + })); + + return ( + + { + const value = labelToValue[item.label]; + if (value) onSelect(value); + }} + mode="toggle" + panelTitle={panelTitle} + triggerIcon={} + /> + + ); +} + +// ── Concrete pickers ─────────────────────────────────────────────────────── + +export const MeshOutputTypePicker = (props: { + options: string[]; + current?: string; + onSelect: (value: string) => void; +}) => ( + +); + +export const PolygonTypePicker = (props: { + options: string[]; + current?: string; + onSelect: (value: string) => void; +}) => ( + +); + +export const GeometryQualityPicker = (props: { + options: string[]; + current?: string; + onSelect: (value: string) => void; +}) => ( + +); + +export const TextureQualityPicker = (props: { + options: string[]; + current?: string; + onSelect: (value: string) => void; +}) => ( + +); + +// Standard two-value quality option lists ("standard" / "detailed"). +export const QUALITY_OPTIONS = ["standard", "detailed"]; + +export const FaceCountPicker = ({ + current, + onSelect, +}: { + current?: number; + onSelect: (value: number | undefined) => void; +}) => { + const items: PopoverItem[] = [ + { label: "Auto", selected: current == null }, + ...FACE_COUNT_PRESETS.map((n) => ({ + label: `${n / 1000}k`, + selected: current === n, + })), + ]; + + return ( + + { + if (item.label === "Auto") { + onSelect(undefined); + return; + } + const n = parseFloat(item.label) * 1000; + if (!Number.isNaN(n)) onSelect(n); + }} + mode="toggle" + panelTitle="Face count" + triggerIcon={} + /> + + ); +}; diff --git a/frontend/apps/artcraft-webapp/src/pages/create-object/create-object-store.ts b/frontend/apps/artcraft-webapp/src/pages/create-object/create-object-store.ts new file mode 100644 index 0000000000..736e8e3d62 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-object/create-object-store.ts @@ -0,0 +1,169 @@ +import { create } from "zustand"; +import { persist } from "zustand/middleware"; +import type { RecreatePayload } from "../../lib/recreate"; +import type { RefImage } from "../../components/prompt-box"; + +// A generated 3D asset (mesh). Shape mirrors the image store's GeneratedImage so +// the shared polling helpers can be reused; `cdn_url` here is the .glb file. +export interface GeneratedAsset { + media_token: string; + cdn_url: string; + maybe_thumbnail_template?: string; +} + +export type MeshBatch = { + id: string; + prompt: string; + status: "pending" | "complete" | "failed"; + assets: GeneratedAsset[]; + createdAt: number; + modelLabel: string; + jobToken?: string; + failureReason?: string; +}; + +export type MeshUiState = { + selectedModelId: string | null; + prompt: string; + meshOutputType: string | undefined; + polygonType: string | undefined; + faceCount: number | undefined; + enablePbr: boolean | undefined; + enableTexture: boolean | undefined; + textureQuality: string | undefined; + geometryQuality: string | undefined; +}; + +// Multi-view side inputs + a mesh-to-mesh input file. Not persisted (File +// handles + data URLs aren't serializable / can exceed the storage quota). +export type MeshInputsState = { + frontImage: RefImage | undefined; + backImage: RefImage | undefined; + leftImage: RefImage | undefined; + rightImage: RefImage | undefined; + inputMesh: RefImage | undefined; +}; + +type CreateObjectState = { + batches: MeshBatch[]; + ui: MeshUiState; + referenceImages: RefImage[]; + inputs: MeshInputsState; + pendingRecreate: RecreatePayload | null; + setUi: (patch: Partial) => void; + setReferenceImages: (images: RefImage[]) => void; + setInputs: (patch: Partial) => void; + setPendingRecreate: (payload: RecreatePayload | null) => void; + consumePendingRecreate: () => RecreatePayload | null; + startBatch: (prompt: string, modelLabel: string) => string; + setBatchJobToken: (batchId: string, jobToken: string) => void; + completeBatch: (batchId: string, assets: GeneratedAsset[]) => void; + failBatch: (batchId: string, reason?: string) => void; + dismissBatch: (id: string) => void; + reset: () => void; +}; + +const DEFAULT_UI: MeshUiState = { + selectedModelId: null, + prompt: "", + meshOutputType: undefined, + polygonType: undefined, + faceCount: undefined, + enablePbr: undefined, + enableTexture: undefined, + textureQuality: undefined, + geometryQuality: undefined, +}; + +const DEFAULT_INPUTS: MeshInputsState = { + frontImage: undefined, + backImage: undefined, + leftImage: undefined, + rightImage: undefined, + inputMesh: undefined, +}; + +export const useCreateObjectStore = create()( + persist( + (set, get) => ({ + batches: [], + ui: { ...DEFAULT_UI }, + referenceImages: [], + inputs: { ...DEFAULT_INPUTS }, + pendingRecreate: null, + + setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })), + + setReferenceImages: (images) => set({ referenceImages: images }), + + setInputs: (patch) => set((s) => ({ inputs: { ...s.inputs, ...patch } })), + + setPendingRecreate: (payload) => set({ pendingRecreate: payload }), + + consumePendingRecreate: () => { + const payload = get().pendingRecreate; + if (payload) set({ pendingRecreate: null }); + return payload; + }, + + startBatch: (prompt, modelLabel) => { + const id = crypto.randomUUID(); + const batch: MeshBatch = { + id, + prompt, + status: "pending", + assets: [], + createdAt: Date.now(), + modelLabel, + }; + set((s) => ({ batches: [...s.batches, batch] })); + return id; + }, + + setBatchJobToken: (batchId, jobToken) => { + set((s) => ({ + batches: s.batches.map((b) => + b.id === batchId ? { ...b, jobToken } : b, + ), + })); + }, + + completeBatch: (batchId, assets) => { + set((s) => ({ + batches: s.batches.map((b) => + b.id === batchId + ? { ...b, status: "complete" as const, assets } + : b, + ), + })); + }, + + failBatch: (batchId, reason) => { + set((s) => ({ + batches: s.batches.map((b) => + b.id === batchId + ? { ...b, status: "failed" as const, failureReason: reason } + : b, + ), + })); + }, + + dismissBatch: (id) => { + set((s) => ({ batches: s.batches.filter((b) => b.id !== id) })); + }, + + reset: () => set({ batches: [] }), + }), + { + name: "artcraft-object-prompt", + // Persist only the lightweight settings; reference images / input files + // are excluded (not serializable, can exceed the localStorage quota). + partialize: (state) => ({ ui: state.ui }), + merge: (persisted, current) => { + const persistedUi = (persisted as { ui?: Partial } | null) + ?.ui; + return { ...current, ui: { ...current.ui, ...(persistedUi ?? {}) } }; + }, + }, + ), +); diff --git a/frontend/apps/artcraft-webapp/src/pages/create-object/create-object.tsx b/frontend/apps/artcraft-webapp/src/pages/create-object/create-object.tsx new file mode 100644 index 0000000000..460dc36a40 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-object/create-object.tsx @@ -0,0 +1,708 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { FilterMediaClasses } from "@storyteller/api"; +import type { OmniGenMeshModelInfo } from "@storyteller/api"; +import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover"; +import { Tooltip } from "@storyteller/ui-tooltip"; +import { Button, ToggleButton } from "@storyteller/ui-button"; +import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal"; +import { + faSparkles, + faGem, + faImage, +} from "@fortawesome/pro-solid-svg-icons"; +import { + PromptBox, + ImagePromptRow, + MobilePromptForm, + MobileSelectField, + type RefImage, +} from "../../components/prompt-box"; +import { + GenerationGallery, + useGalleryData, + useGenerationJobs, + useAuthCheck, + usePromptHeight, + useLightboxNav, + CreateMediaPageShell, +} from "../../components/generation-gallery"; +import { Lightbox } from "../../components/lightbox/lightbox"; +import { is3DModelUrl } from "../../components/lightbox/shared"; +import { useCreateObjectStore } from "./create-object-store"; +import { enqueueMeshGeneration, startPolling } from "./generate-object-api"; +import { + MeshOutputTypePicker, + PolygonTypePicker, + GeometryQualityPicker, + TextureQualityPicker, + FaceCountPicker, + QUALITY_OPTIONS, +} from "./components/MeshOptionPickers"; +import { MeshInputsRow, type MultiViewSlot } from "./components/MeshInputsRow"; +import { useMeshCostEstimate } from "../../lib/cost-estimate-api"; +import { resolveModelOption } from "../../lib/resolve-model-setting"; +import { + useOmniGenMeshModels, + OMNI_GENERATE_OUTAGE_MESSAGE, +} from "@storyteller/omni-gen"; +import { + getCreatorIconPathForModelId, + getModelDescription, + getModelInfo, +} from "@storyteller/model-list"; +import { toast } from "../../components/toast/toast"; +import { useSignupCta } from "../../components/signup-cta-modal"; +import { useInsufficientCredits } from "../../components/insufficient-credits-modal"; + +// ── Constants ──────────────────────────────────────────────────────────── + +const DEFAULT_MODEL_ID = "hunyuan_3d_3"; +const DIMENSIONAL_FILTER = [FilterMediaClasses.DIMENSIONAL]; + +let _modelLookup = new Map(); + +function buildModelPopoverItems( + models: OmniGenMeshModelInfo[], + selectedId: string, +): PopoverItem[] { + _modelLookup = new Map(models.map((m) => [m.model, m])); + return models.map((model) => ({ + label: model.full_name || model.model, + selected: model.model === selectedId, + description: getModelDescription(model.model, model.extra_info_short), + info: getModelInfo(model.model, model.extra_info) || undefined, + icon: ( + {`${model.model} + ), + action: model.model, + })); +} + +// ── Component ──────────────────────────────────────────────────────────── + +export default function CreateObject() { + const { user, authChecked } = useAuthCheck(); + const { loggedIn, openSignupCta } = useSignupCta(); + const openInsufficientCredits = useInsufficientCredits(); + const { promptBoxRef, promptHeight } = usePromptHeight(); + + const { models: apiModels } = useOmniGenMeshModels(); + + const ui = useCreateObjectStore((s) => s.ui); + const setUi = useCreateObjectStore((s) => s.setUi); + + const selectedModel = useMemo((): OmniGenMeshModelInfo | undefined => { + if (!apiModels.length) return undefined; + if (ui.selectedModelId) { + return ( + apiModels.find((m) => m.model === ui.selectedModelId) ?? + apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? + apiModels[0] + ); + } + return apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0]; + }, [apiModels, ui.selectedModelId]); + + const prompt = ui.prompt; + const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]); + + // Capability flags for the selected model. + const supportsText = !!selectedModel?.text_prompt_supported; + const supportsImage = + !!selectedModel?.image_input_supported || + !!selectedModel?.sketch_input_supported; + const supportsMultiView = !!selectedModel?.multi_view_supported; + const supportsMeshInput = !!selectedModel?.mesh_input_supported; + const outputTypes = selectedModel?.mesh_output_types ?? []; + const polygonTypes = selectedModel?.polygon_types ?? []; + const supportsGeometryQuality = !!selectedModel?.geometry_quality_supported; + const supportsTextureQuality = !!selectedModel?.texture_quality_supported; + const supportsFaceCount = !!selectedModel?.face_count_supported; + const supportsPbr = !!selectedModel?.pbr_supported; + const supportsTextureToggle = !!selectedModel?.texture_toggle_supported; + + // Settings are sticky across model switches, resolved against the current + // model's supported options at read time (see lib/resolve-model-setting). + const meshOutputType = resolveModelOption(ui.meshOutputType, outputTypes, outputTypes[0]); + const polygonType = resolveModelOption(ui.polygonType, polygonTypes, polygonTypes[0]); + const geometryQuality = resolveModelOption( + ui.geometryQuality, + supportsGeometryQuality ? QUALITY_OPTIONS : [], + "standard", + ); + const textureQuality = resolveModelOption( + ui.textureQuality, + supportsTextureQuality ? QUALITY_OPTIONS : [], + "standard", + ); + // Toggles default on when supported unless the user turned them off. + const enablePbr = supportsPbr ? (ui.enablePbr ?? false) : false; + const enableTexture = supportsTextureToggle ? (ui.enableTexture ?? true) : true; + + const [isGenerating, setIsGenerating] = useState(false); + const referenceImages = useCreateObjectStore((s) => s.referenceImages); + const setReferenceImages = useCreateObjectStore((s) => s.setReferenceImages); + const inputs = useCreateObjectStore((s) => s.inputs); + const setInputs = useCreateObjectStore((s) => s.setInputs); + // The library picker is shared: it can target the primary reference image or + // any one of the multi-view slots. `null` = closed. + const [libraryTarget, setLibraryTarget] = useState< + null | "primary" | MultiViewSlot + >(null); + const [pickerSelectedIds, setPickerSelectedIds] = useState([]); + + useEffect(() => { + if (libraryTarget) setPickerSelectedIds([]); + }, [libraryTarget]); + + // Batch store + const startBatch = useCreateObjectStore((s) => s.startBatch); + const setBatchJobToken = useCreateObjectStore((s) => s.setBatchJobToken); + const completeBatch = useCreateObjectStore((s) => s.completeBatch); + const failBatch = useCreateObjectStore((s) => s.failBatch); + const dismissBatch = useCreateObjectStore((s) => s.dismissBatch); + const pollingCleanupsRef = useRef void>>(new Map()); + + // Jobs + gallery (only this page's own — mesh objects, not splats) + const meshModelIds = useMemo( + () => apiModels.map((m) => m.model), + [apiModels], + ); + const jobs = useGenerationJobs({ mediaType: "object", enabled: !!user }); + const gallery = useGalleryData({ + username: user?.username ?? null, + filterMediaClasses: DIMENSIONAL_FILTER, + filterModelIds: meshModelIds, + excludeUploads: true, + }); + + // The library's dimensional feed can include a 3D model's cover screenshot as + // its own entry (the backend surfaces it as a "dimensional" media file even + // though its asset is a .png). Keep only real 3D-model files so that stray + // screenshot doesn't show up as a separate card. + const modelGalleryItems = useMemo( + () => gallery.items.filter((i) => !!i.fullImage && is3DModelUrl(i.fullImage)), + [gallery.items], + ); + + // A freshly-completed 3D job has no cover screenshot in its payload (the + // backend renders it a moment later), so its card would be blank. Once the + // library re-fetch returns the same model with its cover, adopt that + // thumbnail so the card stops showing the placeholder. + const galleryById = useMemo( + () => new Map(modelGalleryItems.map((i) => [i.id, i])), + [modelGalleryItems], + ); + const enrichedNewlyCompleted = useMemo( + () => + jobs.newlyCompleted.map((item) => { + if (!item.thumbnail) { + const g = galleryById.get(item.id); + if (g?.thumbnail) return { ...item, thumbnail: g.thumbnail }; + } + return item; + }), + [jobs.newlyCompleted, galleryById], + ); + + const newlyCompletedTokens = useMemo( + () => new Set(enrichedNewlyCompleted.map((i) => i.id)), + [enrichedNewlyCompleted], + ); + + const flatItems = useMemo(() => { + const filtered = modelGalleryItems.filter( + (i) => !newlyCompletedTokens.has(i.id), + ); + return [...enrichedNewlyCompleted, ...filtered]; + }, [enrichedNewlyCompleted, modelGalleryItems, newlyCompletedTokens]); + + // Re-fetch the library shortly after a 3D generation completes so blank + // just-finished cards can pick up their cover screenshot once it's attached. + const galleryRefreshRef = useRef(gallery.refresh); + galleryRefreshRef.current = gallery.refresh; + useEffect(() => { + if (!jobs.newlyCompleted.some((i) => !i.thumbnail)) return; + const t1 = setTimeout(() => galleryRefreshRef.current(), 4000); + const t2 = setTimeout(() => galleryRefreshRef.current(), 10000); + return () => { + clearTimeout(t1); + clearTimeout(t2); + }; + }, [jobs.newlyCompleted]); + + const lightbox = useLightboxNav(flatItems); + + const estimatedCredits = useMeshCostEstimate({ + model: selectedModel?.model ?? "", + referenceImageCount: referenceImages.length, + hasInputMesh: !!inputs.inputMesh, + meshOutputType: outputTypes.length ? meshOutputType : undefined, + polygonType: polygonTypes.length ? polygonType : undefined, + faceCount: supportsFaceCount ? ui.faceCount : undefined, + enablePbr: supportsPbr ? enablePbr : undefined, + enableTexture: supportsTextureToggle ? enableTexture : undefined, + textureQuality: supportsTextureQuality ? textureQuality : undefined, + geometryQuality: supportsGeometryQuality ? geometryQuality : undefined, + }); + + const modelItems = useMemo( + () => buildModelPopoverItems(apiModels, selectedModel?.model ?? ""), + [apiModels, selectedModel?.model], + ); + + const hasContent = + jobs.inProgress.length > 0 || + jobs.failed.length > 0 || + jobs.newlyCompleted.length > 0 || + modelGalleryItems.length > 0 || + gallery.isInitialLoading; + + const hasAnyImage = + referenceImages.length > 0 || + !!inputs.frontImage || + !!inputs.backImage || + !!inputs.leftImage || + !!inputs.rightImage; + + const canGenerate = + !!selectedModel && + !isGenerating && + (supportsMeshInput + ? !!inputs.inputMesh + : (supportsText && prompt.trim().length > 0) || hasAnyImage); + + // ── Effects ────────────────────────────────────────────────────────────── + + // Resume polling for pending batches (page revisit). + useEffect(() => { + const cleanups = pollingCleanupsRef.current; + const pendingBatches = useCreateObjectStore + .getState() + .batches.filter((b) => b.status === "pending" && b.jobToken); + + for (const batch of pendingBatches) { + if (cleanups.has(batch.id)) continue; + const stop = startPolling( + batch.jobToken!, + (assets) => { + completeBatch(batch.id, assets); + cleanups.delete(batch.id); + window.dispatchEvent(new Event("task-queue-update")); + }, + (reason) => { + failBatch(batch.id, reason); + cleanups.delete(batch.id); + window.dispatchEvent(new Event("task-queue-update")); + }, + ); + cleanups.set(batch.id, stop); + } + + return () => { + cleanups.forEach((stop) => stop()); + cleanups.clear(); + }; + }, [completeBatch, failBatch]); + + // ── Handlers ───────────────────────────────────────────────────────────── + + const handleModelChange = useCallback( + (item: PopoverItem) => { + const model = item.action ? _modelLookup.get(item.action) : undefined; + if (!model) return; + setUi({ selectedModelId: model.model }); + }, + [setUi], + ); + + const handleLibraryImageSelect = useCallback( + (items: GalleryItem[]) => { + const item = items[0]; + if (item && libraryTarget) { + const refImage: RefImage = { + id: Math.random().toString(36).substring(7), + url: item.thumbnail || item.fullImage || "", + fullUrl: item.fullImage || undefined, + file: new File([], "library-image"), + mediaToken: item.id, + }; + if (libraryTarget === "primary") { + setReferenceImages([refImage]); + } else if (libraryTarget === "front") { + setInputs({ frontImage: refImage }); + } else if (libraryTarget === "back") { + setInputs({ backImage: refImage }); + } else if (libraryTarget === "left") { + setInputs({ leftImage: refImage }); + } else { + setInputs({ rightImage: refImage }); + } + } + setLibraryTarget(null); + }, + [libraryTarget, setReferenceImages, setInputs], + ); + + const handleGenerate = useCallback(async () => { + if (!loggedIn) { + openSignupCta(); + return; + } + if (!canGenerate || !selectedModel) return; + + setIsGenerating(true); + const batchId = startBatch( + prompt || selectedModel.full_name || selectedModel.model, + selectedModel.full_name ?? selectedModel.model, + ); + + try { + const referenceImageMediaTokens = supportsImage + ? referenceImages + .map((img) => img.mediaToken) + .filter((t): t is string => typeof t === "string" && t.length > 0) + : undefined; + + const result = await enqueueMeshGeneration({ + model: selectedModel.model, + prompt: supportsText && prompt.trim() ? prompt.trim() : undefined, + referenceImageMediaTokens: referenceImageMediaTokens?.length + ? referenceImageMediaTokens + : undefined, + frontImageMediaToken: supportsMultiView + ? inputs.frontImage?.mediaToken + : undefined, + backImageMediaToken: supportsMultiView + ? inputs.backImage?.mediaToken + : undefined, + leftImageMediaToken: supportsMultiView + ? inputs.leftImage?.mediaToken + : undefined, + rightImageMediaToken: supportsMultiView + ? inputs.rightImage?.mediaToken + : undefined, + inputMeshMediaToken: supportsMeshInput + ? inputs.inputMesh?.mediaToken + : undefined, + meshOutputType: outputTypes.length ? meshOutputType : undefined, + polygonType: polygonTypes.length ? polygonType : undefined, + faceCount: supportsFaceCount ? ui.faceCount : undefined, + enablePbr: supportsPbr ? enablePbr : undefined, + enableTexture: supportsTextureToggle ? enableTexture : undefined, + textureQuality: supportsTextureQuality ? textureQuality : undefined, + geometryQuality: supportsGeometryQuality ? geometryQuality : undefined, + }); + + if (!result.success || !result.jobToken) { + if (result.errorCode === 402) { + dismissBatch(batchId); + openInsufficientCredits(); + } else { + if (result.errorCode != null && result.errorCode >= 500) { + toast.error(OMNI_GENERATE_OUTAGE_MESSAGE); + } + failBatch(batchId, result.error ?? "Failed to start generation"); + } + setIsGenerating(false); + return; + } + + setBatchJobToken(batchId, result.jobToken); + window.dispatchEvent(new Event("credits-change")); + window.dispatchEvent(new Event("task-queue-update")); + + const stopPolling = startPolling( + result.jobToken, + (assets) => { + completeBatch(batchId, assets); + pollingCleanupsRef.current.delete(batchId); + window.dispatchEvent(new Event("task-queue-update")); + }, + (reason) => { + failBatch(batchId, reason); + pollingCleanupsRef.current.delete(batchId); + window.dispatchEvent(new Event("task-queue-update")); + }, + ); + pollingCleanupsRef.current.set(batchId, stopPolling); + } catch { + failBatch(batchId, "Network error - please try again"); + } finally { + setIsGenerating(false); + } + }, [ + loggedIn, + openSignupCta, + openInsufficientCredits, + canGenerate, + selectedModel, + prompt, + supportsText, + supportsImage, + supportsMultiView, + supportsMeshInput, + referenceImages, + inputs, + outputTypes, + meshOutputType, + polygonTypes, + polygonType, + supportsFaceCount, + ui.faceCount, + supportsPbr, + enablePbr, + supportsTextureToggle, + enableTexture, + supportsTextureQuality, + textureQuality, + supportsGeometryQuality, + geometryQuality, + startBatch, + setBatchJobToken, + completeBatch, + failBatch, + dismissBatch, + ]); + + const placeholder = supportsText + ? "Describe the 3D object you want..." + : supportsMeshInput + ? "Upload a mesh file below to process" + : "Add a reference image below"; + + // Only render the extra-inputs row when the model actually supports one of its + // slots, otherwise it renders empty but still flags a "row above" the prompt + // box (flat-top visual glitch). + const meshInputsRow = + supportsMultiView || supportsMeshInput ? ( + + ) : undefined; + + // ── Option controls (shared desktop + mobile) ────────────────────────── + + const optionControls = ( + <> + {outputTypes.length > 0 && ( + setUi({ meshOutputType: v })} + /> + )} + {polygonTypes.length > 0 && ( + setUi({ polygonType: v })} + /> + )} + {supportsGeometryQuality && ( + setUi({ geometryQuality: v })} + /> + )} + {supportsTextureQuality && ( + setUi({ textureQuality: v })} + /> + )} + {supportsFaceCount && ( + setUi({ faceCount: v })} + /> + )} + {supportsPbr && ( + + setUi({ enablePbr: !enablePbr })} + /> + + )} + {supportsTextureToggle && ( + + setUi({ enableTexture: !enableTexture })} + /> + + )} + + ); + + // ── Mobile form ───────────────────────────────────────────────────────── + + const mobileForm = ( + + } + frames={ + supportsImage ? ( + setLibraryTarget("primary")} + /> + ) : undefined + } + settingsFields={meshInputsRow} + /> + ); + + // ── Render ──────────────────────────────────────────────────────────── + + return ( + + Sign up to create + + ) + } + bottomOffset={promptHeight + 24} + modelItems={modelItems} + onModelChange={handleModelChange} + promptForm={mobileForm} + gridContent={ + + } + promptBox={ +
+ setLibraryTarget("primary")} + mediaReferenceRow={meshInputsRow} + modelSelector={ + + + } + /> + + } + leftToolbar={optionControls} + /> +
+ } + modals={ + <> + setLibraryTarget(null)} + selectedItemIds={pickerSelectedIds} + onSelectItem={(id) => setPickerSelectedIds([id])} + maxSelections={1} + onUseSelected={handleLibraryImageSelect} + forceFilter="image" + hideFilter + /> + + + } + /> + ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/create-object/generate-object-api.ts b/frontend/apps/artcraft-webapp/src/pages/create-object/generate-object-api.ts new file mode 100644 index 0000000000..d623a9c8bd --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-object/generate-object-api.ts @@ -0,0 +1,196 @@ +import { JobsApi, MediaFilesApi, OmniGenApi } from "@storyteller/api"; +import type { OmniGenMeshRequest } from "@storyteller/api"; +import type { GeneratedAsset } from "./create-object-store"; + +// ── Request params ─────────────────────────────────────────────────────── + +export interface GenerateMeshParams { + model: string; + prompt?: string; + referenceImageMediaTokens?: string[]; + frontImageMediaToken?: string; + backImageMediaToken?: string; + leftImageMediaToken?: string; + rightImageMediaToken?: string; + inputMeshMediaToken?: string; + meshOutputType?: string; + polygonType?: string; + faceCount?: number; + enablePbr?: boolean; + enableTexture?: boolean; + textureQuality?: string; + geometryQuality?: string; +} + +// ── Enqueue generation ─────────────────────────────────────────────────── + +export async function enqueueMeshGeneration(params: GenerateMeshParams): Promise<{ + success: boolean; + jobToken?: string; + error?: string; + errorCode?: number; +}> { + const body: OmniGenMeshRequest = { + model: params.model, + idempotency_token: crypto.randomUUID(), + prompt: params.prompt ?? null, + reference_image_media_tokens: params.referenceImageMediaTokens?.length + ? params.referenceImageMediaTokens + : null, + front_image_media_token: params.frontImageMediaToken ?? null, + back_image_media_token: params.backImageMediaToken ?? null, + left_image_media_token: params.leftImageMediaToken ?? null, + right_image_media_token: params.rightImageMediaToken ?? null, + input_mesh_media_token: params.inputMeshMediaToken ?? null, + mesh_output_type: params.meshOutputType ?? null, + polygon_type: params.polygonType ?? null, + face_count: params.faceCount ?? null, + enable_pbr: params.enablePbr ?? null, + enable_texture: params.enableTexture ?? null, + texture_quality: params.textureQuality ?? null, + geometry_quality: params.geometryQuality ?? null, + }; + + try { + const api = new OmniGenApi(); + const response = await api.generateMesh(body); + if (response.success && response.inference_job_token) { + return { success: true, jobToken: response.inference_job_token }; + } + return { success: false, error: "Generation failed" }; + } catch (err: any) { + return { + success: false, + error: err.message ?? "Request failed", + errorCode: parseHttpStatusCode(err), + }; + } +} + +// Pull the HTTP status out of an ApiManager error (thrown as +// `Error("HTTP error! status: 402")`) so callers can special-case 402. +function parseHttpStatusCode(err: unknown): number | undefined { + const message = err instanceof Error ? err.message : String(err); + const match = /status:\s*(\d+)/.exec(message); + return match ? Number(match[1]) : undefined; +} + +// ── Poll for completion ────────────────────────────────────────────────── + +async function pollJobResult( + jobToken: string, +): Promise<{ status: "pending" | "complete" | "failed"; assets: GeneratedAsset[]; error?: string }> { + const jobsApi = new JobsApi(); + const response = await jobsApi.GetJobByToken({ token: jobToken }); + + if (!response.success || !response.data) { + return { status: "pending", assets: [] }; + } + + const state = response.data; + const statusStr = state.status?.status?.toLowerCase() ?? ""; + + if (statusStr === "complete_success" || statusStr === "complete") { + const result = state.maybe_result as Record | undefined; + const mediaLinks = (result as any)?.media_links; + const entityToken = (result as any)?.entity_token as string | undefined; + + if (mediaLinks?.cdn_url) { + const asset: GeneratedAsset = { + media_token: entityToken ?? jobToken, + cdn_url: mediaLinks.cdn_url, + maybe_thumbnail_template: mediaLinks.maybe_thumbnail_template, + }; + return { status: "complete", assets: [asset] }; + } + + // Rare: no media link on the result payload. Confirm the media file exists. + if (entityToken) { + try { + const mediaApi = new MediaFilesApi(); + const mediaFile = await mediaApi.GetMediaFileByToken({ + mediaFileToken: entityToken, + }); + const cdnUrl = (mediaFile.data as any)?.media_links?.cdn_url; + if (cdnUrl) { + return { + status: "complete", + assets: [{ media_token: entityToken, cdn_url: cdnUrl }], + }; + } + } catch { + // fall through + } + } + + return { status: "complete", assets: [] }; + } + + if ( + statusStr.includes("fail") || + statusStr.includes("error") || + statusStr === "dead" + ) { + return { + status: "failed", + assets: [], + error: + state.status?.maybe_failure_message ?? + state.status?.maybe_extra_status_description ?? + "Generation failed", + }; + } + + return { status: "pending", assets: [] }; +} + +// ── Polling controller ─────────────────────────────────────────────────── + +// 3D generations can run a while, so allow a longer window than images. +export function startPolling( + jobToken: string, + onComplete: (assets: GeneratedAsset[]) => void, + onError: (reason: string) => void, + intervalMs = 4000, + maxAttempts = 150, +): () => void { + let attempts = 0; + let stopped = false; + + const poll = async () => { + if (stopped) return; + attempts++; + + try { + const result = await pollJobResult(jobToken); + if (stopped) return; + + if (result.status === "complete") { + onComplete(result.assets); + return; + } + if (result.status === "failed") { + onError(result.error ?? "Generation failed"); + return; + } + if (attempts >= maxAttempts) { + onError("Generation timed out"); + return; + } + + setTimeout(poll, intervalMs); + } catch { + if (!stopped && attempts < maxAttempts) { + setTimeout(poll, intervalMs * 2); + } else if (!stopped) { + onError("Network error during polling"); + } + } + }; + + setTimeout(poll, 2000); + + return () => { + stopped = true; + }; +} diff --git a/frontend/apps/artcraft-webapp/src/pages/create-object/index.ts b/frontend/apps/artcraft-webapp/src/pages/create-object/index.ts new file mode 100644 index 0000000000..39467e3978 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-object/index.ts @@ -0,0 +1 @@ +export { default } from "./create-object"; diff --git a/frontend/apps/artcraft-webapp/src/pages/create-world/components/ReferenceVideoSlot.tsx b/frontend/apps/artcraft-webapp/src/pages/create-world/components/ReferenceVideoSlot.tsx new file mode 100644 index 0000000000..f6e6838f52 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-world/components/ReferenceVideoSlot.tsx @@ -0,0 +1,108 @@ +import { useRef, useState } from "react"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { + faPlus, + faSpinnerThird, + faXmark, + faVideo, +} from "@fortawesome/pro-solid-svg-icons"; +import { MediaUploadApi, EIntermediateFile } from "@storyteller/api"; +import type { RefVideoAsset } from "../create-world-store"; + +// A single optional reference video for splat generation. Uploads via the +// generic new-video endpoint and stores the returned media token. + +const SLOT_CLASS = + "flex aspect-square w-14 items-center justify-center overflow-hidden rounded-lg border-2 border-dashed border-white/25 bg-white/5 transition-all hover:border-white/40 hover:bg-white/10"; + +export function ReferenceVideoSlot({ + video, + onChange, +}: { + video?: RefVideoAsset; + onChange: (video?: RefVideoAsset) => void; +}) { + const inputRef = useRef(null); + const [uploading, setUploading] = useState(false); + + const handleUpload = async (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (!file) return; + setUploading(true); + const previewUrl = URL.createObjectURL(file); + try { + const api = new MediaUploadApi(); + const response = await api.UploadNewVideo({ + blob: file, + fileName: file.name || `reference-video-${Date.now()}`, + uuid: crypto.randomUUID(), + maybe_title: "splat_reference_video", + is_intermediate_system_file: EIntermediateFile.false, + }); + if (response?.success && response.data) { + onChange({ + id: Math.random().toString(36).substring(7), + previewUrl, + mediaToken: response.data, + }); + } else { + URL.revokeObjectURL(previewUrl); + } + } catch { + URL.revokeObjectURL(previewUrl); + } finally { + setUploading(false); + if (inputRef.current) inputRef.current.value = ""; + } + }; + + const handleRemove = () => { + if (video?.previewUrl) URL.revokeObjectURL(video.previewUrl); + onChange(undefined); + }; + + return ( + // Title on the left, upload slot right-aligned + top-aligned (matches + // ImagePromptRow). Flat bottom on desktop so it seams into the prompt box. +
+
+
+ + Reference video +
+ Guide the world (optional) +
+ + {video ? ( +
+
+ ) : uploading ? ( +
+ +
+ ) : ( + + )} +
+ ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/create-world/create-world-store.ts b/frontend/apps/artcraft-webapp/src/pages/create-world/create-world-store.ts new file mode 100644 index 0000000000..495994b5e7 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-world/create-world-store.ts @@ -0,0 +1,146 @@ +import { create } from "zustand"; +import { persist } from "zustand/middleware"; +import type { RecreatePayload } from "../../lib/recreate"; +import type { RefImage } from "../../components/prompt-box"; + +// A generated splat (3D world). `cdn_url` here is the .spz file. +export interface GeneratedAsset { + media_token: string; + cdn_url: string; + maybe_thumbnail_template?: string; +} + +export type SplatBatch = { + id: string; + prompt: string; + status: "pending" | "complete" | "failed"; + assets: GeneratedAsset[]; + createdAt: number; + modelLabel: string; + jobToken?: string; + failureReason?: string; +}; + +export type SplatUiState = { + selectedModelId: string | null; + prompt: string; + isPanoramic: boolean | undefined; + disableRecaption: boolean | undefined; +}; + +// A stored reference video: media token + a local object-URL for preview. Not +// persisted (File handles / blob URLs aren't serializable). +export type RefVideoAsset = { + id: string; + previewUrl: string; + mediaToken: string; +}; + +type CreateWorldState = { + batches: SplatBatch[]; + ui: SplatUiState; + referenceImages: RefImage[]; + referenceVideo: RefVideoAsset | undefined; + pendingRecreate: RecreatePayload | null; + setUi: (patch: Partial) => void; + setReferenceImages: (images: RefImage[]) => void; + setReferenceVideo: (video: RefVideoAsset | undefined) => void; + setPendingRecreate: (payload: RecreatePayload | null) => void; + consumePendingRecreate: () => RecreatePayload | null; + startBatch: (prompt: string, modelLabel: string) => string; + setBatchJobToken: (batchId: string, jobToken: string) => void; + completeBatch: (batchId: string, assets: GeneratedAsset[]) => void; + failBatch: (batchId: string, reason?: string) => void; + dismissBatch: (id: string) => void; + reset: () => void; +}; + +const DEFAULT_UI: SplatUiState = { + selectedModelId: null, + prompt: "", + isPanoramic: undefined, + disableRecaption: undefined, +}; + +export const useCreateWorldStore = create()( + persist( + (set, get) => ({ + batches: [], + ui: { ...DEFAULT_UI }, + referenceImages: [], + referenceVideo: undefined, + pendingRecreate: null, + + setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })), + + setReferenceImages: (images) => set({ referenceImages: images }), + + setReferenceVideo: (video) => set({ referenceVideo: video }), + + setPendingRecreate: (payload) => set({ pendingRecreate: payload }), + + consumePendingRecreate: () => { + const payload = get().pendingRecreate; + if (payload) set({ pendingRecreate: null }); + return payload; + }, + + startBatch: (prompt, modelLabel) => { + const id = crypto.randomUUID(); + const batch: SplatBatch = { + id, + prompt, + status: "pending", + assets: [], + createdAt: Date.now(), + modelLabel, + }; + set((s) => ({ batches: [...s.batches, batch] })); + return id; + }, + + setBatchJobToken: (batchId, jobToken) => { + set((s) => ({ + batches: s.batches.map((b) => + b.id === batchId ? { ...b, jobToken } : b, + ), + })); + }, + + completeBatch: (batchId, assets) => { + set((s) => ({ + batches: s.batches.map((b) => + b.id === batchId + ? { ...b, status: "complete" as const, assets } + : b, + ), + })); + }, + + failBatch: (batchId, reason) => { + set((s) => ({ + batches: s.batches.map((b) => + b.id === batchId + ? { ...b, status: "failed" as const, failureReason: reason } + : b, + ), + })); + }, + + dismissBatch: (id) => { + set((s) => ({ batches: s.batches.filter((b) => b.id !== id) })); + }, + + reset: () => set({ batches: [] }), + }), + { + name: "artcraft-world-prompt", + partialize: (state) => ({ ui: state.ui }), + merge: (persisted, current) => { + const persistedUi = (persisted as { ui?: Partial } | null) + ?.ui; + return { ...current, ui: { ...current.ui, ...(persistedUi ?? {}) } }; + }, + }, + ), +); diff --git a/frontend/apps/artcraft-webapp/src/pages/create-world/create-world.tsx b/frontend/apps/artcraft-webapp/src/pages/create-world/create-world.tsx new file mode 100644 index 0000000000..4fa369421a --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-world/create-world.tsx @@ -0,0 +1,581 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { FilterMediaClasses } from "@storyteller/api"; +import type { OmniGenSplatModelInfo } from "@storyteller/api"; +import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover"; +import { Tooltip } from "@storyteller/ui-tooltip"; +import { Button, ToggleButton } from "@storyteller/ui-button"; +import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal"; +import { + faSparkles, + faPanorama, + faWandMagicSparkles, +} from "@fortawesome/pro-solid-svg-icons"; +import { + PromptBox, + ImagePromptRow, + MobilePromptForm, + MobileSelectField, + type RefImage, +} from "../../components/prompt-box"; +import { + GenerationGallery, + useGalleryData, + useGenerationJobs, + useAuthCheck, + usePromptHeight, + useLightboxNav, + CreateMediaPageShell, +} from "../../components/generation-gallery"; +import { Lightbox } from "../../components/lightbox/lightbox"; +import { is3DModelUrl } from "../../components/lightbox/shared"; +import { useCreateWorldStore } from "./create-world-store"; +import { enqueueSplatGeneration, startPolling } from "./generate-world-api"; +import { ReferenceVideoSlot } from "./components/ReferenceVideoSlot"; +import { useSplatCostEstimate } from "../../lib/cost-estimate-api"; +import { + useOmniGenSplatModels, + OMNI_GENERATE_OUTAGE_MESSAGE, +} from "@storyteller/omni-gen"; +import { + getCreatorIconPathForModelId, + getModelDescription, + getModelInfo, +} from "@storyteller/model-list"; +import { toast } from "../../components/toast/toast"; +import { useSignupCta } from "../../components/signup-cta-modal"; +import { useInsufficientCredits } from "../../components/insufficient-credits-modal"; + +// ── Constants ──────────────────────────────────────────────────────────── + +const DEFAULT_MODEL_ID = "marble_1p1"; +const DIMENSIONAL_FILTER = [FilterMediaClasses.DIMENSIONAL]; + +let _modelLookup = new Map(); + +function buildModelPopoverItems( + models: OmniGenSplatModelInfo[], + selectedId: string, +): PopoverItem[] { + _modelLookup = new Map(models.map((m) => [m.model, m])); + return models.map((model) => ({ + label: model.full_name || model.model, + selected: model.model === selectedId, + description: getModelDescription(model.model, model.extra_info_short), + info: getModelInfo(model.model, model.extra_info) || undefined, + icon: ( + {`${model.model} + ), + action: model.model, + })); +} + +// ── Component ──────────────────────────────────────────────────────────── + +export default function CreateWorld() { + const { user, authChecked } = useAuthCheck(); + const { loggedIn, openSignupCta } = useSignupCta(); + const openInsufficientCredits = useInsufficientCredits(); + const { promptBoxRef, promptHeight } = usePromptHeight(); + + const { models: apiModels } = useOmniGenSplatModels(); + + const ui = useCreateWorldStore((s) => s.ui); + const setUi = useCreateWorldStore((s) => s.setUi); + + const selectedModel = useMemo((): OmniGenSplatModelInfo | undefined => { + if (!apiModels.length) return undefined; + if (ui.selectedModelId) { + return ( + apiModels.find((m) => m.model === ui.selectedModelId) ?? + apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? + apiModels[0] + ); + } + return apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0]; + }, [apiModels, ui.selectedModelId]); + + const prompt = ui.prompt; + const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]); + + const supportsText = !!selectedModel?.text_prompt_supported; + const supportsImage = !!selectedModel?.image_references_supported; + const maxImageRefs = selectedModel?.image_references_max ?? 4; + const supportsVideo = !!selectedModel?.video_reference_supported; + const supportsPanorama = !!selectedModel?.panorama_supported; + const supportsDisableRecaption = !!selectedModel?.disable_recaption_supported; + + const isPanoramic = supportsPanorama ? (ui.isPanoramic ?? false) : false; + const disableRecaption = supportsDisableRecaption + ? (ui.disableRecaption ?? false) + : false; + + const [isGenerating, setIsGenerating] = useState(false); + const referenceImages = useCreateWorldStore((s) => s.referenceImages); + const setReferenceImages = useCreateWorldStore((s) => s.setReferenceImages); + const referenceVideo = useCreateWorldStore((s) => s.referenceVideo); + const setReferenceVideo = useCreateWorldStore((s) => s.setReferenceVideo); + const [isImagePickerOpen, setIsImagePickerOpen] = useState(false); + const [pickerSelectedIds, setPickerSelectedIds] = useState([]); + + useEffect(() => { + if (isImagePickerOpen) setPickerSelectedIds([]); + }, [isImagePickerOpen]); + + const imagePickerMax = Math.max(1, maxImageRefs - referenceImages.length); + const handlePickerSelect = useCallback( + (id: string) => { + setPickerSelectedIds((prev) => { + if (prev.includes(id)) return prev.filter((x) => x !== id); + if (prev.length >= imagePickerMax) { + return imagePickerMax === 1 ? [id] : prev; + } + return [...prev, id]; + }); + }, + [imagePickerMax], + ); + + // Batch store + const startBatch = useCreateWorldStore((s) => s.startBatch); + const setBatchJobToken = useCreateWorldStore((s) => s.setBatchJobToken); + const completeBatch = useCreateWorldStore((s) => s.completeBatch); + const failBatch = useCreateWorldStore((s) => s.failBatch); + const dismissBatch = useCreateWorldStore((s) => s.dismissBatch); + const pollingCleanupsRef = useRef void>>(new Map()); + + // Jobs + gallery (only splats, not mesh objects) + const splatModelIds = useMemo( + () => apiModels.map((m) => m.model), + [apiModels], + ); + const jobs = useGenerationJobs({ mediaType: "splat", enabled: !!user }); + const gallery = useGalleryData({ + username: user?.username ?? null, + filterMediaClasses: DIMENSIONAL_FILTER, + filterModelIds: splatModelIds, + excludeUploads: true, + }); + + // A freshly-completed splat job has no cover screenshot in its payload (the + // backend renders it a moment later), so its card would be blank. Once the + // library re-fetch returns the same world with its cover, adopt that + // thumbnail so the card stops showing the placeholder. + // The library's dimensional feed can include a world's cover screenshot as + // its own entry (surfaced as a "dimensional" media file even though its asset + // is a .png). Keep only real 3D/splat files so that stray screenshot doesn't + // show up as a separate card. + const modelGalleryItems = useMemo( + () => gallery.items.filter((i) => !!i.fullImage && is3DModelUrl(i.fullImage)), + [gallery.items], + ); + + const galleryById = useMemo( + () => new Map(modelGalleryItems.map((i) => [i.id, i])), + [modelGalleryItems], + ); + const enrichedNewlyCompleted = useMemo( + () => + jobs.newlyCompleted.map((item) => { + if (!item.thumbnail) { + const g = galleryById.get(item.id); + if (g?.thumbnail) return { ...item, thumbnail: g.thumbnail }; + } + return item; + }), + [jobs.newlyCompleted, galleryById], + ); + + const newlyCompletedTokens = useMemo( + () => new Set(enrichedNewlyCompleted.map((i) => i.id)), + [enrichedNewlyCompleted], + ); + + const flatItems = useMemo(() => { + const filtered = modelGalleryItems.filter( + (i) => !newlyCompletedTokens.has(i.id), + ); + return [...enrichedNewlyCompleted, ...filtered]; + }, [enrichedNewlyCompleted, modelGalleryItems, newlyCompletedTokens]); + + // Re-fetch the library shortly after a generation completes so blank + // just-finished cards can pick up their cover screenshot once it's attached. + const galleryRefreshRef = useRef(gallery.refresh); + galleryRefreshRef.current = gallery.refresh; + useEffect(() => { + if (!jobs.newlyCompleted.some((i) => !i.thumbnail)) return; + const t1 = setTimeout(() => galleryRefreshRef.current(), 4000); + const t2 = setTimeout(() => galleryRefreshRef.current(), 10000); + return () => { + clearTimeout(t1); + clearTimeout(t2); + }; + }, [jobs.newlyCompleted]); + + const lightbox = useLightboxNav(flatItems); + + const estimatedCredits = useSplatCostEstimate({ + model: selectedModel?.model ?? "", + referenceImageCount: referenceImages.length, + hasReferenceVideo: !!referenceVideo, + isPanoramic: supportsPanorama ? isPanoramic : undefined, + disableRecaption: supportsDisableRecaption ? disableRecaption : undefined, + }); + + const modelItems = useMemo( + () => buildModelPopoverItems(apiModels, selectedModel?.model ?? ""), + [apiModels, selectedModel?.model], + ); + + const hasContent = + jobs.inProgress.length > 0 || + jobs.failed.length > 0 || + jobs.newlyCompleted.length > 0 || + modelGalleryItems.length > 0 || + gallery.isInitialLoading; + + const canGenerate = + !!selectedModel && + !isGenerating && + (prompt.trim().length > 0 || referenceImages.length > 0 || !!referenceVideo); + + // ── Effects ────────────────────────────────────────────────────────────── + + useEffect(() => { + const cleanups = pollingCleanupsRef.current; + const pendingBatches = useCreateWorldStore + .getState() + .batches.filter((b) => b.status === "pending" && b.jobToken); + + for (const batch of pendingBatches) { + if (cleanups.has(batch.id)) continue; + const stop = startPolling( + batch.jobToken!, + (assets) => { + completeBatch(batch.id, assets); + cleanups.delete(batch.id); + window.dispatchEvent(new Event("task-queue-update")); + }, + (reason) => { + failBatch(batch.id, reason); + cleanups.delete(batch.id); + window.dispatchEvent(new Event("task-queue-update")); + }, + ); + cleanups.set(batch.id, stop); + } + + return () => { + cleanups.forEach((stop) => stop()); + cleanups.clear(); + }; + }, [completeBatch, failBatch]); + + // ── Handlers ───────────────────────────────────────────────────────────── + + const handleModelChange = useCallback( + (item: PopoverItem) => { + const model = item.action ? _modelLookup.get(item.action) : undefined; + if (!model) return; + setUi({ selectedModelId: model.model }); + }, + [setUi], + ); + + const handleLibraryImageSelect = useCallback( + (items: GalleryItem[]) => { + const availableSlots = Math.max(0, maxImageRefs - referenceImages.length); + const newImages: RefImage[] = items + .slice(0, availableSlots) + .map((item) => ({ + id: Math.random().toString(36).substring(7), + url: item.thumbnail || item.fullImage || "", + fullUrl: item.fullImage || undefined, + file: new File([], "library-image"), + mediaToken: item.id, + })); + setReferenceImages([...referenceImages, ...newImages]); + setIsImagePickerOpen(false); + }, + [maxImageRefs, referenceImages, setReferenceImages], + ); + + const handleGenerate = useCallback(async () => { + if (!loggedIn) { + openSignupCta(); + return; + } + if (!canGenerate || !selectedModel) return; + + setIsGenerating(true); + const batchId = startBatch( + prompt || selectedModel.full_name || selectedModel.model, + selectedModel.full_name ?? selectedModel.model, + ); + + try { + const referenceImageMediaTokens = supportsImage + ? referenceImages + .map((img) => img.mediaToken) + .filter((t): t is string => typeof t === "string" && t.length > 0) + : undefined; + + const result = await enqueueSplatGeneration({ + model: selectedModel.model, + prompt: supportsText && prompt.trim() ? prompt.trim() : undefined, + referenceImageMediaTokens: referenceImageMediaTokens?.length + ? referenceImageMediaTokens + : undefined, + referenceVideoMediaToken: supportsVideo + ? referenceVideo?.mediaToken + : undefined, + isPanoramic: supportsPanorama ? isPanoramic : undefined, + disableRecaption: supportsDisableRecaption ? disableRecaption : undefined, + }); + + if (!result.success || !result.jobToken) { + if (result.errorCode === 402) { + dismissBatch(batchId); + openInsufficientCredits(); + } else { + if (result.errorCode != null && result.errorCode >= 500) { + toast.error(OMNI_GENERATE_OUTAGE_MESSAGE); + } + failBatch(batchId, result.error ?? "Failed to start generation"); + } + setIsGenerating(false); + return; + } + + setBatchJobToken(batchId, result.jobToken); + window.dispatchEvent(new Event("credits-change")); + window.dispatchEvent(new Event("task-queue-update")); + + const stopPolling = startPolling( + result.jobToken, + (assets) => { + completeBatch(batchId, assets); + pollingCleanupsRef.current.delete(batchId); + window.dispatchEvent(new Event("task-queue-update")); + }, + (reason) => { + failBatch(batchId, reason); + pollingCleanupsRef.current.delete(batchId); + window.dispatchEvent(new Event("task-queue-update")); + }, + ); + pollingCleanupsRef.current.set(batchId, stopPolling); + } catch { + failBatch(batchId, "Network error - please try again"); + } finally { + setIsGenerating(false); + } + }, [ + loggedIn, + openSignupCta, + openInsufficientCredits, + canGenerate, + selectedModel, + prompt, + supportsText, + supportsImage, + supportsVideo, + supportsPanorama, + supportsDisableRecaption, + referenceImages, + referenceVideo, + isPanoramic, + disableRecaption, + startBatch, + setBatchJobToken, + completeBatch, + failBatch, + dismissBatch, + ]); + + // ── Option controls (shared desktop + mobile) ────────────────────────── + + const optionControls = ( + <> + {supportsPanorama && ( + + setUi({ isPanoramic: !isPanoramic })} + /> + + )} + {supportsDisableRecaption && ( + + setUi({ disableRecaption: !disableRecaption })} + /> + + )} + + ); + + const videoRow = supportsVideo ? ( + + ) : undefined; + + // ── Mobile form ───────────────────────────────────────────────────────── + + const mobileForm = ( + + } + frames={ + supportsImage ? ( + setIsImagePickerOpen(true)} + /> + ) : undefined + } + settingsFields={videoRow} + /> + ); + + // ── Render ──────────────────────────────────────────────────────────── + + return ( + + Sign up to create + + ) + } + bottomOffset={promptHeight + 24} + modelItems={modelItems} + onModelChange={handleModelChange} + promptForm={mobileForm} + gridContent={ + + } + promptBox={ +
+ setIsImagePickerOpen(true)} + mediaReferenceRow={videoRow} + modelSelector={ + + + } + /> + + } + leftToolbar={optionControls} + /> +
+ } + modals={ + <> + setIsImagePickerOpen(false)} + selectedItemIds={pickerSelectedIds} + onSelectItem={handlePickerSelect} + maxSelections={imagePickerMax} + onUseSelected={handleLibraryImageSelect} + forceFilter="image" + hideFilter + /> + + + } + /> + ); +} diff --git a/frontend/apps/artcraft-webapp/src/pages/create-world/generate-world-api.ts b/frontend/apps/artcraft-webapp/src/pages/create-world/generate-world-api.ts new file mode 100644 index 0000000000..8d7740d0de --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-world/generate-world-api.ts @@ -0,0 +1,181 @@ +import { JobsApi, MediaFilesApi, OmniGenApi } from "@storyteller/api"; +import type { OmniGenSplatRequest } from "@storyteller/api"; +import type { GeneratedAsset } from "./create-world-store"; + +// ── Request params ─────────────────────────────────────────────────────── + +export interface GenerateSplatParams { + model: string; + prompt?: string; + referenceImageMediaTokens?: string[]; + referenceVideoMediaToken?: string; + isPanoramic?: boolean; + disableRecaption?: boolean; +} + +// ── Enqueue generation ─────────────────────────────────────────────────── + +export async function enqueueSplatGeneration( + params: GenerateSplatParams, +): Promise<{ + success: boolean; + jobToken?: string; + error?: string; + errorCode?: number; +}> { + const body: OmniGenSplatRequest = { + model: params.model, + idempotency_token: crypto.randomUUID(), + prompt: params.prompt ?? null, + reference_image_media_tokens: params.referenceImageMediaTokens?.length + ? params.referenceImageMediaTokens + : null, + reference_video_media_token: params.referenceVideoMediaToken ?? null, + is_panoramic: params.isPanoramic ?? null, + disable_recaption: params.disableRecaption ?? null, + }; + + try { + const api = new OmniGenApi(); + const response = await api.generateSplat(body); + if (response.success && response.inference_job_token) { + return { success: true, jobToken: response.inference_job_token }; + } + return { success: false, error: "Generation failed" }; + } catch (err: any) { + return { + success: false, + error: err.message ?? "Request failed", + errorCode: parseHttpStatusCode(err), + }; + } +} + +function parseHttpStatusCode(err: unknown): number | undefined { + const message = err instanceof Error ? err.message : String(err); + const match = /status:\s*(\d+)/.exec(message); + return match ? Number(match[1]) : undefined; +} + +// ── Poll for completion ────────────────────────────────────────────────── + +async function pollJobResult( + jobToken: string, +): Promise<{ status: "pending" | "complete" | "failed"; assets: GeneratedAsset[]; error?: string }> { + const jobsApi = new JobsApi(); + const response = await jobsApi.GetJobByToken({ token: jobToken }); + + if (!response.success || !response.data) { + return { status: "pending", assets: [] }; + } + + const state = response.data; + const statusStr = state.status?.status?.toLowerCase() ?? ""; + + if (statusStr === "complete_success" || statusStr === "complete") { + const result = state.maybe_result as Record | undefined; + const mediaLinks = (result as any)?.media_links; + const entityToken = (result as any)?.entity_token as string | undefined; + + if (mediaLinks?.cdn_url) { + return { + status: "complete", + assets: [ + { + media_token: entityToken ?? jobToken, + cdn_url: mediaLinks.cdn_url, + maybe_thumbnail_template: mediaLinks.maybe_thumbnail_template, + }, + ], + }; + } + + if (entityToken) { + try { + const mediaApi = new MediaFilesApi(); + const mediaFile = await mediaApi.GetMediaFileByToken({ + mediaFileToken: entityToken, + }); + const cdnUrl = (mediaFile.data as any)?.media_links?.cdn_url; + if (cdnUrl) { + return { + status: "complete", + assets: [{ media_token: entityToken, cdn_url: cdnUrl }], + }; + } + } catch { + // fall through + } + } + + return { status: "complete", assets: [] }; + } + + if ( + statusStr.includes("fail") || + statusStr.includes("error") || + statusStr === "dead" + ) { + return { + status: "failed", + assets: [], + error: + state.status?.maybe_failure_message ?? + state.status?.maybe_extra_status_description ?? + "Generation failed", + }; + } + + return { status: "pending", assets: [] }; +} + +// ── Polling controller ─────────────────────────────────────────────────── + +// Splat / world generation is slow, so allow a generous window. +export function startPolling( + jobToken: string, + onComplete: (assets: GeneratedAsset[]) => void, + onError: (reason: string) => void, + intervalMs = 5000, + maxAttempts = 180, +): () => void { + let attempts = 0; + let stopped = false; + + const poll = async () => { + if (stopped) return; + attempts++; + + try { + const result = await pollJobResult(jobToken); + if (stopped) return; + + if (result.status === "complete") { + onComplete(result.assets); + return; + } + if (result.status === "failed") { + onError(result.error ?? "Generation failed"); + return; + } + if (attempts >= maxAttempts) { + onError("Generation timed out"); + return; + } + + setTimeout(poll, intervalMs); + } catch { + if (!stopped && attempts < maxAttempts) { + setTimeout(poll, intervalMs * 2); + } else if (!stopped) { + onError("Network error during polling"); + } + } + }; + + setTimeout(poll, 2000); + + return () => { + stopped = true; + }; +} diff --git a/frontend/apps/artcraft-webapp/src/pages/create-world/index.ts b/frontend/apps/artcraft-webapp/src/pages/create-world/index.ts new file mode 100644 index 0000000000..2b64241c01 --- /dev/null +++ b/frontend/apps/artcraft-webapp/src/pages/create-world/index.ts @@ -0,0 +1 @@ +export { default } from "./create-world"; diff --git a/frontend/apps/artcraft-webapp/src/pages/home/home.tsx b/frontend/apps/artcraft-webapp/src/pages/home/home.tsx index 9d902d766a..2afba51477 100644 --- a/frontend/apps/artcraft-webapp/src/pages/home/home.tsx +++ b/frontend/apps/artcraft-webapp/src/pages/home/home.tsx @@ -7,6 +7,7 @@ import { faMusic, faWandMagicSparkles, faCube, + faGlobe, faFilm, faObjectGroup, faArrowRight, @@ -57,6 +58,24 @@ const APPS: AppCard[] = [ iconBg: "bg-pink-500/25 border-pink-400/30", iconColor: "text-pink-300", }, + { + label: "3D Object", + description: "Turn a prompt or image into a 3D model.", + href: "/create-object", + icon: faCube, + accent: "from-cyan-500/20 to-cyan-500/0", + iconBg: "bg-cyan-500/25 border-cyan-400/30", + iconColor: "text-cyan-300", + }, + { + label: "3D World", + description: "Turn a prompt or image into an explorable world.", + href: "/create-world", + icon: faGlobe, + accent: "from-teal-500/20 to-teal-500/0", + iconBg: "bg-teal-500/25 border-teal-400/30", + iconColor: "text-teal-300", + }, { label: "Edit 3D", description: "Compose 3D scenes and render with AI cameras.", diff --git a/frontend/apps/artcraft-webapp/src/pages/library/library.tsx b/frontend/apps/artcraft-webapp/src/pages/library/library.tsx index 3ca3a619e5..54f1f76ad2 100644 --- a/frontend/apps/artcraft-webapp/src/pages/library/library.tsx +++ b/frontend/apps/artcraft-webapp/src/pages/library/library.tsx @@ -23,7 +23,7 @@ import { FOLDER_DROP_EVENT, type GalleryItem, } from "@storyteller/ui-gallery-modal"; -import { PLACEHOLDER_IMAGES } from "@storyteller/common"; +import { PLACEHOLDER_IMAGES, is3DModelUrl } from "@storyteller/common"; import { showActionReminder, isActionReminderOpen, @@ -285,7 +285,14 @@ export default function Library() { if (response.success && response.data) { const newItems = response.data .filter( - (item: any) => item.media_type !== FilterMediaType.SCENE_JSON, + (item: any) => + item.media_type !== FilterMediaType.SCENE_JSON && + // Drop 3D-model cover screenshots the backend surfaces as + // "dimensional" items whose asset is actually a .png. + !( + item.media_class === "dimensional" && + !is3DModelUrl(item.media_links?.cdn_url) + ), ) .map(mapRawToGalleryItem); setAllItems((prev) => (reset ? newItems : [...prev, ...newItems])); diff --git a/frontend/apps/artcraft-webapp/tailwind.config.js b/frontend/apps/artcraft-webapp/tailwind.config.js index 65f0345839..19f7ad2739 100644 --- a/frontend/apps/artcraft-webapp/tailwind.config.js +++ b/frontend/apps/artcraft-webapp/tailwind.config.js @@ -46,6 +46,7 @@ module.exports = { green: { ...colors.green, DEFAULT: "#308752" }, emerald: colors.emerald, teal: colors.teal, + cyan: colors.cyan, yellow: colors.yellow, pink: colors.pink, purple: colors.purple, diff --git a/frontend/apps/artcraft/app/public/resources/images/services/hyper3d.svg b/frontend/apps/artcraft/app/public/resources/images/services/hyper3d.svg new file mode 100644 index 0000000000..aa99495e5f --- /dev/null +++ b/frontend/apps/artcraft/app/public/resources/images/services/hyper3d.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/apps/artcraft/app/public/resources/images/services/meshy.svg b/frontend/apps/artcraft/app/public/resources/images/services/meshy.svg new file mode 100644 index 0000000000..da77f22eb2 --- /dev/null +++ b/frontend/apps/artcraft/app/public/resources/images/services/meshy.svg @@ -0,0 +1 @@ +Meshy \ No newline at end of file diff --git a/frontend/apps/artcraft/app/public/resources/images/services/tripo.svg b/frontend/apps/artcraft/app/public/resources/images/services/tripo.svg new file mode 100644 index 0000000000..40f4c37190 --- /dev/null +++ b/frontend/apps/artcraft/app/public/resources/images/services/tripo.svg @@ -0,0 +1 @@ +Tripo \ No newline at end of file diff --git a/frontend/libs/api/src/lib/OmniGenApi.ts b/frontend/libs/api/src/lib/OmniGenApi.ts index 65fd8e3010..0910cebe3e 100644 --- a/frontend/libs/api/src/lib/OmniGenApi.ts +++ b/frontend/libs/api/src/lib/OmniGenApi.ts @@ -304,6 +304,137 @@ export interface OmniGenProviderEntry { models: OmniGenProviderModelEntry[]; } +// ── Mesh (3D object) request / response types ──────────────────────────── + +// Shared body for POST /v1/omni_gen/cost/mesh and /v1/omni_gen/generate/mesh. +export interface OmniGenMeshRequest { + // Required despite being nullable (backend rejects when absent). + model: string; + idempotency_token?: string | null; + prompt?: string | null; + // Primary/front input image, or the sketch image for sketch-to-3D. + reference_image_media_tokens?: string[] | null; + // Multi-view input (one image per side). + front_image_media_token?: string | null; + back_image_media_token?: string | null; + left_image_media_token?: string | null; + right_image_media_token?: string | null; + // Input mesh file for mesh-to-mesh models (part splitting, retopology). + input_mesh_media_token?: string | null; + // "normal" | "low_poly" | "geometry" + mesh_output_type?: string | null; + // "triangle" | "quad" + polygon_type?: string | null; + face_count?: number | null; + enable_pbr?: boolean | null; + enable_texture?: boolean | null; + // "standard" | "detailed" + texture_quality?: string | null; + geometry_quality?: string | null; +} + +export interface OmniGenMeshCostResponse { + success: boolean; + cost_in_credits?: number | null; + cost_in_usd_cents?: number | null; + has_watermark: boolean; + is_free: boolean; + is_rate_limited: boolean; + is_unlimited: boolean; + failures_are_refunded?: boolean | null; +} + +export interface OmniGenMeshGenerateResponse { + success: boolean; + inference_job_token: string; + all_job_tokens: string[]; +} + +export interface OmniGenMeshModelInfo { + model: string; + // ModelCreator enum as a snake_case string, e.g. "tencent". + model_creator?: string | null; + full_name?: string | null; + extra_info_short?: string | null; + extra_info?: string | null; + text_prompt_supported?: boolean | null; + image_input_supported?: boolean | null; + sketch_input_supported?: boolean | null; + multi_view_supported?: boolean | null; + mesh_input_supported?: boolean | null; + // CommonMeshOutputType values: "normal" | "low_poly" | "geometry". + mesh_output_types?: string[] | null; + // CommonPolygonType values: "triangle" | "quad". + polygon_types?: string[] | null; + face_count_supported?: boolean | null; + pbr_supported?: boolean | null; + texture_toggle_supported?: boolean | null; + texture_quality_supported?: boolean | null; + geometry_quality_supported?: boolean | null; + is_disabled?: boolean | null; +} + +export interface OmniGenMeshModelsResponse { + success: boolean; + models: OmniGenMeshModelInfo[]; + providers: OmniGenProviderEntry[]; +} + +// ── Splat (3D world) request / response types ──────────────────────────── + +// Shared body for POST /v1/omni_gen/cost/splat and /v1/omni_gen/generate/splat. +export interface OmniGenSplatRequest { + // Required despite being nullable (backend rejects when absent). + model: string; + idempotency_token?: string | null; + prompt?: string | null; + // Multi-view input of the same world. + reference_image_media_tokens?: string[] | null; + reference_video_media_token?: string | null; + // Whether the single reference image is a 360-degree panorama. + is_panoramic?: boolean | null; + disable_recaption?: boolean | null; +} + +export interface OmniGenSplatCostResponse { + success: boolean; + cost_in_credits?: number | null; + cost_in_usd_cents?: number | null; + has_watermark: boolean; + is_free: boolean; + is_rate_limited: boolean; + is_unlimited: boolean; + failures_are_refunded?: boolean | null; +} + +export interface OmniGenSplatGenerateResponse { + success: boolean; + inference_job_token: string; + all_job_tokens: string[]; +} + +export interface OmniGenSplatModelInfo { + model: string; + // ModelCreator enum as a snake_case string, e.g. "world_labs". + model_creator?: string | null; + full_name?: string | null; + extra_info_short?: string | null; + extra_info?: string | null; + text_prompt_supported?: boolean | null; + image_references_supported?: boolean | null; + image_references_max?: number | null; + video_reference_supported?: boolean | null; + panorama_supported?: boolean | null; + disable_recaption_supported?: boolean | null; + is_disabled?: boolean | null; +} + +export interface OmniGenSplatModelsResponse { + success: boolean; + models: OmniGenSplatModelInfo[]; + providers: OmniGenProviderEntry[]; +} + // ── Helpers ────────────────────────────────────────────────────────────── /** Strip keys whose value is null or undefined so the server only sees @@ -347,6 +478,26 @@ export class OmniGenApi extends ApiManager { }); } + public async getMeshModels( + provider?: string, + ): Promise { + const query = provider ? { provider } : undefined; + return this.get({ + endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/models/mesh`, + query, + }); + } + + public async getSplatModels( + provider?: string, + ): Promise { + const query = provider ? { provider } : undefined; + return this.get({ + endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/models/splat`, + query, + }); + } + // ── Cost estimates ─────────────────────────────────────────────────── public async estimateImageCost( @@ -376,6 +527,24 @@ export class OmniGenApi extends ApiManager { }); } + public async estimateMeshCost( + body: OmniGenMeshRequest, + ): Promise { + return this.post, OmniGenMeshCostResponse>({ + endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/cost/mesh`, + body: stripNulls(body), + }); + } + + public async estimateSplatCost( + body: OmniGenSplatRequest, + ): Promise { + return this.post, OmniGenSplatCostResponse>({ + endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/cost/splat`, + body: stripNulls(body), + }); + } + // ── Generation ─────────────────────────────────────────────────────── public async generateImage( @@ -404,4 +573,22 @@ export class OmniGenApi extends ApiManager { body: stripNulls(body), }); } + + public async generateMesh( + body: OmniGenMeshRequest, + ): Promise { + return this.post, OmniGenMeshGenerateResponse>({ + endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/generate/mesh`, + body: stripNulls(body), + }); + } + + public async generateSplat( + body: OmniGenSplatRequest, + ): Promise { + return this.post, OmniGenSplatGenerateResponse>({ + endpoint: `${this.getApiSchemeAndHost()}/v1/omni_gen/generate/splat`, + body: stripNulls(body), + }); + } } diff --git a/frontend/libs/common/src/lib/utils/thumbnail-utils.ts b/frontend/libs/common/src/lib/utils/thumbnail-utils.ts index 7bb6811cac..28afc2fa51 100644 --- a/frontend/libs/common/src/lib/utils/thumbnail-utils.ts +++ b/frontend/libs/common/src/lib/utils/thumbnail-utils.ts @@ -143,3 +143,16 @@ export function getPlaceholderForMediaClass(mediaClass?: string): string { return PLACEHOLDER_IMAGES.DEFAULT; } } + +// File extensions for actual 3D-model assets (meshes + gaussian splats). +export const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz"]; + +/** Whether a URL points at a real 3D-model asset (vs. e.g. a cover screenshot). + * Used to keep 3D-model cover images — which the backend can surface as + * "dimensional" media files whose asset is actually a .png — out of the 3D + * feeds and library. */ +export function is3DModelUrl(url: string | null | undefined): boolean { + if (!url) return false; + const lower = url.toLowerCase(); + return MODEL_3D_EXTENSIONS.some((ext) => lower.includes(ext)); +} diff --git a/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx b/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx index beae2467c8..091a684de4 100644 --- a/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx +++ b/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx @@ -41,6 +41,7 @@ import { getMediaThumbnail, THUMBNAIL_SIZES, PLACEHOLDER_IMAGES, + is3DModelUrl, } from "@storyteller/common"; import { galleryModalDraggingUnder, @@ -883,7 +884,14 @@ export const GalleryModal = React.memo( setItemsLoadError(null); const newItems = response.data .filter( - (item: any) => item.media_type !== FilterMediaType.SCENE_JSON, + (item: any) => + item.media_type !== FilterMediaType.SCENE_JSON && + // Drop 3D-model cover screenshots the backend surfaces as + // "dimensional" items whose asset is actually a .png. + !( + item.media_class === "dimensional" && + !is3DModelUrl(item.media_links?.cdn_url) + ), ) .map(mapApiItem); diff --git a/frontend/libs/components/generation-list/src/lib/types.ts b/frontend/libs/components/generation-list/src/lib/types.ts index 1999e5fed5..0ce99603ce 100644 --- a/frontend/libs/components/generation-list/src/lib/types.ts +++ b/frontend/libs/components/generation-list/src/lib/types.ts @@ -1,7 +1,11 @@ // Canonical data shapes for the merged generation feed (in-progress / failed / // completed). Hosts (webapp, desktop) map their own job sources into these. -export type GenerationMediaClass = "image" | "video" | "audio"; +export type GenerationMediaClass = + | "image" + | "video" + | "audio" + | "dimensional"; /** Plural noun for batch captions, e.g. "Generating 4 images". */ export function batchNoun(mediaClass: GenerationMediaClass): string { @@ -10,6 +14,8 @@ export function batchNoun(mediaClass: GenerationMediaClass): string { return "videos"; case "audio": return "audio clips"; + case "dimensional": + return "3D models"; default: return "images"; } diff --git a/frontend/libs/components/generation-list/src/lib/useGalleryData.ts b/frontend/libs/components/generation-list/src/lib/useGalleryData.ts index 1f47c94041..a4ca4996b4 100644 --- a/frontend/libs/components/generation-list/src/lib/useGalleryData.ts +++ b/frontend/libs/components/generation-list/src/lib/useGalleryData.ts @@ -27,12 +27,32 @@ const getLabel = (item: any) => { // ── Hook ─────────────────────────────────────────────────────────────────── +// Pull a cover image / screenshot URL off a dimensional (3D) library item. The +// item's own cdn_url is the .glb/.spz asset, so 3D cards render the separate +// cover image the backend attaches instead. Mirrors the gallery-modal logic. +function get3DCoverThumbnail(item: any): string | null { + const cover = item?.cover_image; + if (!cover) return null; + return ( + cover.maybe_links?.maybe_thumbnail_template?.replace("{WIDTH}", "512") ?? + cover.maybe_links?.cdn_url ?? + cover.maybe_cover_image_public_bucket_url ?? + null + ); +} + export function useGalleryData(options: { username: string | null; filterMediaClasses: FilterMediaClasses[]; excludeUploads?: boolean; + // When set, dimensional (3D) items are kept only if their model belongs to + // this list. The library API filters by media class ("dimensional") but can't + // tell mesh objects from splat worlds apart, so the two 3D create pages pass + // their own model ids to keep each feed showing only its own product. + filterModelIds?: string[]; }) { - const { username, filterMediaClasses, excludeUploads } = options; + const { username, filterMediaClasses, excludeUploads, filterModelIds } = + options; const [items, setItems] = useState([]); const [isLoading, setIsLoading] = useState(false); @@ -44,15 +64,16 @@ export function useGalleryData(options: { const api = useMemo(() => new GalleryModalApi(), []); const mapApiItem = useCallback((item: any): GalleryItem => { - // 3D and audio media have no image thumbnail — their cards render a - // placeholder icon / waveform player instead. - const hasImageThumbnail = - item.media_class !== "dimensional" && item.media_class !== "audio"; - const thumbnail = hasImageThumbnail - ? getMediaThumbnail(item.media_links, item.media_class, { - size: THUMBNAIL_SIZES.LARGE, - }) - : null; + // 3D uses the cover image / screenshot; audio has no thumbnail (its card + // renders a waveform player); everything else uses the media thumbnail. + const thumbnail = + item.media_class === "audio" + ? null + : item.media_class === "dimensional" + ? get3DCoverThumbnail(item) + : getMediaThumbnail(item.media_links, item.media_class, { + size: THUMBNAIL_SIZES.LARGE, + }); return { id: item.token, @@ -85,11 +106,25 @@ export function useGalleryData(options: { }); if (response.success && response.data) { + const modelIdSet = + filterModelIds && filterModelIds.length + ? new Set(filterModelIds) + : null; + const newItems = response.data .filter( (item: any) => item.media_type !== FilterMediaType.SCENE_JSON && - !(excludeUploads && item.origin_category === "upload"), + !(excludeUploads && item.origin_category === "upload") && + // Split 3D history by model so the object and world pages don't + // show each other's generations. Non-dimensional items and items + // without a known model are left untouched. + !( + modelIdSet && + item.media_class === "dimensional" && + item.maybe_model_type && + !modelIdSet.has(item.maybe_model_type) + ), ) .map(mapApiItem); @@ -112,7 +147,15 @@ export function useGalleryData(options: { setIsInitialLoading(false); isLoadingRef.current = false; }, - [username, filterMediaClasses, pageIndex, api, mapApiItem, excludeUploads], + [ + username, + filterMediaClasses, + pageIndex, + api, + mapApiItem, + excludeUploads, + filterModelIds, + ], ); // Initial load + filter change. When logged out (no username), clear the @@ -129,7 +172,8 @@ export function useGalleryData(options: { setHasMore(true); setIsInitialLoading(true); loadItems(true); - }, [username, JSON.stringify(filterMediaClasses)]); // eslint-disable-line react-hooks/exhaustive-deps + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [username, JSON.stringify(filterMediaClasses), JSON.stringify(filterModelIds)]); const loadMore = useCallback(() => { if (hasMore && !isLoadingRef.current) { diff --git a/frontend/libs/components/generation-list/src/lib/useGenerationJobs.ts b/frontend/libs/components/generation-list/src/lib/useGenerationJobs.ts index 99e12eb6ec..3f3419c5fc 100644 --- a/frontend/libs/components/generation-list/src/lib/useGenerationJobs.ts +++ b/frontend/libs/components/generation-list/src/lib/useGenerationJobs.ts @@ -50,6 +50,7 @@ const DEFAULT_DURATION_MS: Record = { image: 30000, video: 900000, audio: 120000, + dimensional: 120000, }; // Cache per-task durations @@ -57,11 +58,24 @@ const taskDurationCache = new Map(); // ── Helpers ──────────────────────────────────────────────────────────────── +// The backend `inference_category` strings for 3D are "object_generation" +// (mesh) and "splat_generation" (gaussian splats). Both render as the +// "dimensional" media class, but stay distinct here so the two 3D create pages +// can each filter to only their own generations. +export type JobMediaFilter = + | "image" + | "video" + | "audio" + | "object" + | "splat"; + function getJobMediaType( job: Job, -): "image" | "video" | "audio" | "other" { +): JobMediaFilter | "other" { const cat = job.request.inference_category?.toLowerCase() ?? ""; if (cat.includes("video")) return "video"; + if (cat.includes("object")) return "object"; + if (cat.includes("splat") || cat.includes("gaussian")) return "splat"; if (cat.includes("image")) return "image"; if (cat.includes("audio")) return "audio"; return "other"; @@ -69,9 +83,25 @@ function getJobMediaType( function getJobMediaClass(job: Job): GenerationMediaClass { const inferred = getJobMediaType(job); + if (inferred === "object" || inferred === "splat") return "dimensional"; return inferred === "other" ? "image" : inferred; } +// Pull a cover image / screenshot URL off a 3D result (job result or a batch +// media file). The raw cdn_url is the .glb/.spz asset itself, so 3D cards use +// the separate cover image the backend attaches; falls back to null (the card +// then shows its 3D placeholder). +function get3DCoverThumbnail(source: any): string | null { + const cover = source?.cover_image; + if (!cover) return null; + return ( + cover.maybe_links?.maybe_thumbnail_template?.replace("{WIDTH}", "512") ?? + cover.maybe_links?.cdn_url ?? + cover.maybe_cover_image_public_bucket_url ?? + null + ); +} + function getModelLabel(job: Job, promptsMap?: Map): string { const promptToken = job.request.maybe_prompt_token; const cachedPrompt = promptToken @@ -204,13 +234,16 @@ function jobToGalleryItem( if (!result?.entity_token) return null; const mediaClass = getJobMediaClass(job); - // Audio has no image thumbnail — its card renders a waveform player. + // Audio has no image thumbnail — its card renders a waveform player. 3D uses + // the result's cover image / screenshot (the cdn_url is the mesh/splat file). const thumbnail = mediaClass === "audio" ? null - : getMediaThumbnail(result.media_links, mediaClass, { - size: THUMBNAIL_SIZES.LARGE, - }); + : mediaClass === "dimensional" + ? get3DCoverThumbnail(result) + : getMediaThumbnail(result.media_links, mediaClass, { + size: THUMBNAIL_SIZES.LARGE, + }); return { id: result.entity_token, @@ -249,9 +282,11 @@ async function expandBatchItems( const thumbnail = item.mediaClass === "audio" ? null - : getMediaThumbnail(file.media_links, item.mediaClass, { - size: THUMBNAIL_SIZES.LARGE, - }) || cdnUrl; + : item.mediaClass === "dimensional" + ? get3DCoverThumbnail(file) + : getMediaThumbnail(file.media_links, item.mediaClass, { + size: THUMBNAIL_SIZES.LARGE, + }) || cdnUrl; return { id: file.token, label: item.label, @@ -273,7 +308,7 @@ async function expandBatchItems( // ── Hook ─────────────────────────────────────────────────────────────────── export function useGenerationJobs(options: { - mediaType: GenerationMediaClass; + mediaType: JobMediaFilter; enabled?: boolean; }) { const { mediaType, enabled = true } = options; diff --git a/frontend/libs/components/tooltip/src/lib/tooltip.tsx b/frontend/libs/components/tooltip/src/lib/tooltip.tsx index 4eae74fdff..6c6db6bbbf 100644 --- a/frontend/libs/components/tooltip/src/lib/tooltip.tsx +++ b/frontend/libs/components/tooltip/src/lib/tooltip.tsx @@ -159,10 +159,16 @@ export const Tooltip = ({ let current = triggerRef.current.parentElement; while (current && current !== document.body) { const cs = getComputedStyle(current); + // backdrop-filter also establishes a containing block for position:fixed + // in Chromium/WebKit (our glass surfaces use backdrop-blur), so a fixed + // tooltip inside one would otherwise be positioned/clipped incorrectly. + const backdropFilter = + cs.backdropFilter || (cs as unknown as Record)["webkitBackdropFilter"]; if ( cs.transform !== "none" || cs.willChange === "transform" || - (cs.filter && cs.filter !== "none") + (cs.filter && cs.filter !== "none") || + (backdropFilter && backdropFilter !== "none") ) { const r = current.getBoundingClientRect(); return { x: r.left, y: r.top }; diff --git a/frontend/libs/model-list/src/lib/classes/metadata/ModelCreatorIconForId.ts b/frontend/libs/model-list/src/lib/classes/metadata/ModelCreatorIconForId.ts index 0e1988d90f..2ab669e1e9 100644 --- a/frontend/libs/model-list/src/lib/classes/metadata/ModelCreatorIconForId.ts +++ b/frontend/libs/model-list/src/lib/classes/metadata/ModelCreatorIconForId.ts @@ -25,6 +25,19 @@ export const MODEL_ID_PREFIX_CREATORS: Array<[string, ModelCreator]> = [ // Beeble SwitchX (background change) has no provider icon — use the ArtCraft // mark since it's surfaced as an ArtCraft feature. ["switch_x", ModelCreator.ArtCraft], + // 3D mesh / splat models (not in ALL_MODELS_LIST). + ["hunyuan", ModelCreator.Tencent], + ["marble", ModelCreator.WorldLabs], +]; + +// Direct model-id-prefix → services icon filename, for 3D brands that don't +// have a ModelCreator enum value yet. Drop the matching .svg into +// public/images/services/ (webapp) and resources/images/services/ (desktop). +const MODEL_ID_PREFIX_ICON_FILES: Array<[string, string]> = [ + ["tripo", "tripo.svg"], + ["meshy", "meshy.svg"], + // Rodin is Hyper3D's model. + ["rodin", "hyper3d.svg"], ]; /** @@ -36,8 +49,12 @@ export const getCreatorIconPathForModelId = (modelId: string): string => { (m) => m.id === modelId || m.tauriId === modelId, ); if (model) return getCreatorIconPath(model.creator); + const base = getServicesBasePath(); + for (const [prefix, file] of MODEL_ID_PREFIX_ICON_FILES) { + if (modelId.startsWith(prefix)) return `${base}/${file}`; + } for (const [prefix, creator] of MODEL_ID_PREFIX_CREATORS) { if (modelId.startsWith(prefix)) return getCreatorIconPath(creator); } - return `${getServicesBasePath()}/generic.svg`; + return `${base}/generic.svg`; }; diff --git a/frontend/libs/omni-gen/src/lib/omni-gen-hooks.ts b/frontend/libs/omni-gen/src/lib/omni-gen-hooks.ts index 41fc0db977..bad68e5180 100644 --- a/frontend/libs/omni-gen/src/lib/omni-gen-hooks.ts +++ b/frontend/libs/omni-gen/src/lib/omni-gen-hooks.ts @@ -4,6 +4,8 @@ import type { OmniGenAudioModelDetails, OmniGenImageModelInfo, OmniGenVideoModelInfo, + OmniGenMeshModelInfo, + OmniGenSplatModelInfo, } from "@storyteller/api"; // Shown when the OmniGen service is unavailable (e.g. returns 5xx). Separate @@ -44,6 +46,16 @@ let audioModelsFetching = false; let audioModelsError: string | null = null; let audioModelsListeners: Array<() => void> = []; +let meshModelsCache: OmniGenMeshModelInfo[] | null = null; +let meshModelsFetching = false; +let meshModelsError: string | null = null; +let meshModelsListeners: Array<() => void> = []; + +let splatModelsCache: OmniGenSplatModelInfo[] | null = null; +let splatModelsFetching = false; +let splatModelsError: string | null = null; +let splatModelsListeners: Array<() => void> = []; + function notifyImageListeners() { imageModelsListeners.forEach((cb) => cb()); imageModelsListeners = []; @@ -59,6 +71,16 @@ function notifyAudioListeners() { audioModelsListeners = []; } +function notifyMeshListeners() { + meshModelsListeners.forEach((cb) => cb()); + meshModelsListeners = []; +} + +function notifySplatListeners() { + splatModelsListeners.forEach((cb) => cb()); + splatModelsListeners = []; +} + function fetchImageModelsOnce() { if (imageModelsCache || imageModelsFetching) return; imageModelsFetching = true; @@ -140,6 +162,60 @@ function fetchAudioModelsOnce() { ); } +function fetchMeshModelsOnce() { + if (meshModelsCache || meshModelsFetching) return; + meshModelsFetching = true; + meshModelsError = null; + const api = new OmniGenApi(); + api.getMeshModels().then( + (res) => { + meshModelsFetching = false; + if (res.success && res.models?.length) { + meshModelsCache = res.models.filter((m) => m.is_disabled !== true); + } else { + meshModelsError = "No mesh models returned from API"; + console.warn("[OmniGen] Mesh models response:", res); + errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE); + } + notifyMeshListeners(); + }, + (err) => { + meshModelsFetching = false; + meshModelsError = err?.message ?? "Failed to fetch mesh models"; + console.error("[OmniGen] Failed to fetch mesh models:", err); + errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE); + notifyMeshListeners(); + }, + ); +} + +function fetchSplatModelsOnce() { + if (splatModelsCache || splatModelsFetching) return; + splatModelsFetching = true; + splatModelsError = null; + const api = new OmniGenApi(); + api.getSplatModels().then( + (res) => { + splatModelsFetching = false; + if (res.success && res.models?.length) { + splatModelsCache = res.models.filter((m) => m.is_disabled !== true); + } else { + splatModelsError = "No splat models returned from API"; + console.warn("[OmniGen] Splat models response:", res); + errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE); + } + notifySplatListeners(); + }, + (err) => { + splatModelsFetching = false; + splatModelsError = err?.message ?? "Failed to fetch splat models"; + console.error("[OmniGen] Failed to fetch splat models:", err); + errorNotifier(OMNI_MODELS_OUTAGE_MESSAGE); + notifySplatListeners(); + }, + ); +} + // ── Hooks ──────────────────────────────────────────────────────────────── export function useOmniGenImageModels(): { @@ -267,3 +343,85 @@ export function useOmniGenAudioModels(): { return { models, isLoading, error }; } + +export function useOmniGenMeshModels(): { + models: OmniGenMeshModelInfo[]; + isLoading: boolean; + error: string | null; +} { + const [models, setModels] = useState( + meshModelsCache ?? [], + ); + const [isLoading, setIsLoading] = useState(!meshModelsCache); + const [error, setError] = useState(meshModelsError); + + useEffect(() => { + if (meshModelsCache) { + setModels(meshModelsCache); + setIsLoading(false); + setError(null); + return; + } + + const onReady = () => { + setIsLoading(false); + if (meshModelsCache) { + setModels(meshModelsCache); + setError(null); + } else { + setError(meshModelsError); + } + }; + + meshModelsListeners.push(onReady); + fetchMeshModelsOnce(); + + return () => { + meshModelsListeners = meshModelsListeners.filter((cb) => cb !== onReady); + }; + }, []); + + return { models, isLoading, error }; +} + +export function useOmniGenSplatModels(): { + models: OmniGenSplatModelInfo[]; + isLoading: boolean; + error: string | null; +} { + const [models, setModels] = useState( + splatModelsCache ?? [], + ); + const [isLoading, setIsLoading] = useState(!splatModelsCache); + const [error, setError] = useState(splatModelsError); + + useEffect(() => { + if (splatModelsCache) { + setModels(splatModelsCache); + setIsLoading(false); + setError(null); + return; + } + + const onReady = () => { + setIsLoading(false); + if (splatModelsCache) { + setModels(splatModelsCache); + setError(null); + } else { + setError(splatModelsError); + } + }; + + splatModelsListeners.push(onReady); + fetchSplatModelsOnce(); + + return () => { + splatModelsListeners = splatModelsListeners.filter( + (cb) => cb !== onReady, + ); + }; + }, []); + + return { models, isLoading, error }; +}