diff --git a/frontend/apps/artcraft-webapp/vite.config.ts b/frontend/apps/artcraft-webapp/vite.config.ts index c19153dc8c..e653514dcd 100644 --- a/frontend/apps/artcraft-webapp/vite.config.ts +++ b/frontend/apps/artcraft-webapp/vite.config.ts @@ -7,6 +7,70 @@ import { nxViteTsPaths } from '@nx/vite/plugins/nx-tsconfig-paths.plugin'; import { execSync } from 'child_process'; import path from 'path'; +// Dev-only CDN proxy. Media assets (splats, GLBs, images) are fetched with +// fetch(), but the CDNs only send CORS headers for the production origin — +// not localhost. The web adapter's fetchAsset rewrites absolute CDN URLs to +// `/__cdn//` in DEV; this middleware forwards them server-side +// (no browser CORS) to whatever host is embedded in the path. +// +// NB: This must be a middleware, not a `server.proxy` entry. The previous +// proxy config used a `router` option to pick the target host from the path, +// but `router` is an http-proxy-middleware feature — Vite's http-proxy +// silently ignores it, so every request went to the hardcoded default target +// and any asset on a different host (e.g. the dev R2 bucket) came back 403. +function cdnProxyPlugin() { + return { + name: 'cdn-proxy', + configureServer(server: { + middlewares: { + use: ( + route: string, + handler: ( + req: import('http').IncomingMessage, + res: import('http').ServerResponse, + ) => void, + ) => void; + }; + }) { + server.middlewares.use('/__cdn', async (req, res) => { + try { + // Connect strips the `/__cdn` mount prefix from req.url. + const match = (req.url ?? '').match(/^\/([^/?]+)(\/[^?]*)?(\?.*)?$/); + if (!match) { + res.statusCode = 400; + res.end('Bad /__cdn path; expected /__cdn//'); + return; + } + const [, host, assetPath = '/', query = ''] = match; + const upstream = await fetch(`https://${host}${assetPath}${query}`, { + method: req.method === 'HEAD' ? 'HEAD' : 'GET', + headers: req.headers.range + ? { range: String(req.headers.range) } + : undefined, + }); + res.statusCode = upstream.status; + for (const header of [ + 'content-type', + 'content-length', + 'accept-ranges', + 'content-range', + 'etag', + 'cache-control', + ]) { + const value = upstream.headers.get(header); + if (value) res.setHeader(header, value); + } + const body = Buffer.from(await upstream.arrayBuffer()); + res.end(body); + } catch (e) { + res.statusCode = 502; + res.end(`CDN proxy error: ${e}`); + } + }); + }, + }; +} + // Custom plugin to generate news.json on dev server start function generateNewsPlugin() { return { @@ -38,29 +102,15 @@ export default defineConfig(() => ({ Origin: 'https://api.storyteller.ai', }, }, - // Dev-only CDN proxy. Media assets (splats, GLBs, images) are fetched - // from the CDN with fetch(), but the CDN only sends CORS headers for the - // production origin — not localhost. The web adapter's fetchAsset rewrites - // absolute CDN URLs to `/__cdn//` in DEV; this forwards them - // server-side (no browser CORS). The target host is taken from the path - // so any cdn-N.fakeyou.com / Cloudflare host works. - '/__cdn': { - target: 'https://cdn-2.fakeyou.com', - changeOrigin: true, - secure: true, - router: (req) => { - const match = req.url?.match(/^\/__cdn\/([^/]+)/); - return match ? `https://${match[1]}` : 'https://cdn-2.fakeyou.com'; - }, - rewrite: (path) => path.replace(/^\/__cdn\/[^/]+/, ''), - }, + // NB: `/__cdn` asset proxying lives in `cdnProxyPlugin` above — it + // needs per-request host routing, which `server.proxy` can't do. }, }, preview:{ port: 4301, host: 'localhost', }, - plugins: [generateNewsPlugin(), nxViteTsPaths(), react(), wasm(), topLevelAwait()], + plugins: [cdnProxyPlugin(), generateNewsPlugin(), nxViteTsPaths(), react(), wasm(), topLevelAwait()], // Uncomment this if you are using workers. // worker: { // plugins: [ nxViteTsPaths() ], diff --git a/frontend/libs/common/src/lib/utils/thumbnail-utils.ts b/frontend/libs/common/src/lib/utils/thumbnail-utils.ts index 28afc2fa51..774c647b00 100644 --- a/frontend/libs/common/src/lib/utils/thumbnail-utils.ts +++ b/frontend/libs/common/src/lib/utils/thumbnail-utils.ts @@ -145,7 +145,7 @@ export function getPlaceholderForMediaClass(mediaClass?: string): string { } // File extensions for actual 3D-model assets (meshes + gaussian splats). -export const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz"]; +export const MODEL_3D_EXTENSIONS = [".glb", ".gltf", ".fbx", ".spz", ".obj", ".ply"]; /** Whether a URL points at a real 3D-model asset (vs. e.g. a cover screenshot). * Used to keep 3D-model cover images — which the backend can surface as diff --git a/frontend/libs/components/gallery-modal/src/lib/GalleryDraggableItem.tsx b/frontend/libs/components/gallery-modal/src/lib/GalleryDraggableItem.tsx index ddd1a1ab20..3a98d92187 100644 --- a/frontend/libs/components/gallery-modal/src/lib/GalleryDraggableItem.tsx +++ b/frontend/libs/components/gallery-modal/src/lib/GalleryDraggableItem.tsx @@ -426,7 +426,9 @@ export const GalleryDraggableItem: React.FC = ({ Drag - {item.mediaClass === "dimensional" + {item.mediaClass === "dimensional" || + item.mediaClass === "mesh" || + item.mediaClass === "splat" ? " to add to scene" : " to add"} diff --git a/frontend/libs/components/gallery-modal/src/lib/GalleryItemMenuItems.tsx b/frontend/libs/components/gallery-modal/src/lib/GalleryItemMenuItems.tsx index fbb2ca835c..b0612bbfba 100644 --- a/frontend/libs/components/gallery-modal/src/lib/GalleryItemMenuItems.tsx +++ b/frontend/libs/components/gallery-modal/src/lib/GalleryItemMenuItems.tsx @@ -73,7 +73,9 @@ export const GalleryItemMenuItems: React.FC = ({ {item.mediaClass === "video" ? "View video" - : item.mediaClass === "dimensional" + : item.mediaClass === "dimensional" || + item.mediaClass === "mesh" || + item.mediaClass === "splat" ? "View 3D" : "View image"} diff --git a/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx b/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx index 091a684de4..c373093aea 100644 --- a/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx +++ b/frontend/libs/components/gallery-modal/src/lib/gallery-modal.tsx @@ -60,6 +60,7 @@ import { faImage, faVideo, faCube, + faGlobe, faUpload, faExpand, faCompress, @@ -88,6 +89,8 @@ interface GalleryCacheEntry { pageIndex: number; hasMore: boolean; timestamp: number; + /** Keyset cursor for the mesh/splat tabs (their endpoints paginate by cursor). */ + cursor?: string; } const galleryCacheMap = new Map(); @@ -304,29 +307,37 @@ const SIDEBAR_FILTERS = [ { id: "all", label: "All Assets", icon: faBorderAll }, { id: "image", label: "Image", icon: faImage }, { id: "video", label: "Video", icon: faVideo }, - { id: "3d", label: "3D", icon: faCube }, + { id: "mesh", label: "Mesh", icon: faCube }, + { id: "splat", label: "Splat", icon: faGlobe }, { id: "uploaded", label: "Uploaded", icon: faUpload }, ]; +// Media classes for the user-list endpoint. The Mesh and Splat tabs don't use +// it — they call the session mesh/splat list endpoints instead. const getFilterMediaClass = (filter: string) => { switch (filter) { case "image": return [FilterMediaClasses.IMAGE]; case "video": return [FilterMediaClasses.VIDEO]; - case "3d": - return [FilterMediaClasses.DIMENSIONAL]; case "uploaded": return [ FilterMediaClasses.IMAGE, FilterMediaClasses.VIDEO, FilterMediaClasses.DIMENSIONAL, + FilterMediaClasses.MESH, + FilterMediaClasses.SPLAT, ]; default: return undefined; } }; +// The backend used to file all 3D media under the (now deprecated) coarse +// "dimensional" class; new records are "mesh" or "splat". Treat all three as 3D. +const is3DClass = (mediaClass: string | undefined | null): boolean => + mediaClass === "dimensional" || mediaClass === "mesh" || mediaClass === "splat"; + const getLabel = (item: any) => { if (!!item.maybe_title) { return item.maybe_title; @@ -337,7 +348,10 @@ const getLabel = (item: any) => { case "video": return "Video Generation"; case "dimensional": + case "mesh": return "3D Object Generation"; + case "splat": + return "3D World Generation"; default: return "Generation"; } @@ -791,6 +805,10 @@ export const GalleryModal = React.memo( [], ); + // Keyset cursors for the mesh/splat tabs (their endpoints paginate by + // cursor rather than page index), keyed by filter id. + const byClassCursorRef = useRef>({}); + // Map raw API item to GalleryItem const mapApiItem = useCallback( (item: any): GalleryItem => ({ @@ -799,7 +817,7 @@ export const GalleryModal = React.memo( thumbnail: item.media_class === "video" ? item.media_links.maybe_video_previews?.animated - : item.media_class === "dimensional" + : is3DClass(item.media_class) ? // Prefer the modern CDN link; fall back to the deprecated // bucket URL for older responses. (item.cover_image?.maybe_links?.cdn_url ?? @@ -815,13 +833,15 @@ export const GalleryModal = React.memo( (item.filter_media_classes ? item.filter_media_classes[0] : "image"), isUpload: item.origin_category === "upload", assetType: - item.media_class === "dimensional" + item.media_class === "dimensional" || item.media_class === "mesh" ? item.maybe_animation_type || item.origin_product_category === "character" || (item.origin && item.origin.product_category === "character") ? "character" : "object" - : undefined, + : item.media_class === "splat" + ? "splat" + : undefined, }), [], ); @@ -834,7 +854,7 @@ export const GalleryModal = React.memo( id: item.token, label: getLabel(item), thumbnail: - item.media_class === "dimensional" + is3DClass(item.media_class) ? (item.cover_image?.maybe_links?.cdn_url ?? item.cover_image?.maybe_cover_image_public_bucket_url ?? null) @@ -864,6 +884,63 @@ export const GalleryModal = React.memo( setLoading(true); if (!reset) setPaginationLoading(true); try { + if (currentFilter === "mesh" || currentFilter === "splat") { + // The Mesh / Splat tabs use the session by-class list endpoints + // (server-scoped, cursor-paginated) instead of the user list. + const cursor = reset + ? undefined + : byClassCursorRef.current[currentFilter]; + const response = + currentFilter === "splat" + ? await mediaFilesApi.ListSessionSplatMediaFiles({ + cursor, + page_size: PAGE_SIZE, + }) + : await mediaFilesApi.ListSessionMeshMediaFiles({ + cursor, + page_size: PAGE_SIZE, + }); + + if (response.success && response.data) { + setItemsLoadError(null); + const newItems = response.data.map(mapApiItem); + + if (reset) { + setAllItems(newItems); + } else { + setAllItems((prev) => [...prev, ...newItems]); + } + + const nextCursor = response.pagination?.maybe_next ?? undefined; + byClassCursorRef.current[currentFilter] = nextCursor; + const more = newItems.length >= PAGE_SIZE && !!nextCursor; + setPageIndex(0); + setHasMore(more); + + const cacheKey = getCacheKey(); + setAllItems((latest) => { + galleryCacheMap.set(cacheKey, { + items: latest, + pageIndex: 0, + hasMore: more, + timestamp: Date.now(), + cursor: nextCursor, + }); + return latest; + }); + } else { + setItemsLoadError( + response.errorMessage || "Request failed (unknown error)", + ); + } + + setLoading(false); + setPaginationLoading(false); + setInitialLoading(false); + isLoadingRef.current = false; + return; + } + const filterMediaClasses = getFilterMediaClass(currentFilter); const query = { filter_media_classes: filterMediaClasses, @@ -871,7 +948,6 @@ export const GalleryModal = React.memo( include_user_uploads: currentFilter === "uploaded" || currentFilter === "all" || - currentFilter === "3d" || currentFilter === "image" || currentFilter === "video", user_uploads_only: currentFilter === "uploaded", @@ -970,7 +1046,7 @@ export const GalleryModal = React.memo( setInitialLoading(false); isLoadingRef.current = false; }, - [api, mapApiItem, getCacheKey], + [api, mediaFilesApi, mapApiItem, getCacheKey], ); // refresh logic — shows cached items immediately, then background-refreshes @@ -985,6 +1061,7 @@ export const GalleryModal = React.memo( setAllItems(cached.items); setPageIndex(cached.pageIndex); setHasMore(cached.hasMore); + byClassCursorRef.current[activeFilterRef.current] = cached.cursor; setInitialLoading(false); // Background refresh — silently fetch page 0 and merge new items loadItems(true); @@ -2004,7 +2081,8 @@ export const GalleryModal = React.memo( if ((item as any).mediaType === "scene_json") return false; if (!showUploads && activeFilter !== "uploaded" && item.isUpload) return false; - if (activeFilter === "3d") return item.mediaClass === "dimensional"; + if (activeFilter === "mesh") return item.mediaClass === "mesh"; + if (activeFilter === "splat") return item.mediaClass === "splat"; if (activeFilter === "image") return item.mediaClass === "image"; if (activeFilter === "video") return item.mediaClass === "video"; if (activeFilter === "all") { @@ -2341,10 +2419,14 @@ export const GalleryModal = React.memo( ? maxSelections === 1 ? "Select Video" : "Select Videos" - : activeFilter === "3d" + : activeFilter === "mesh" ? maxSelections === 1 ? "Select 3D Object" : "Select 3D Objects" + : activeFilter === "splat" + ? maxSelections === 1 + ? "Select 3D World" + : "Select 3D Worlds" : activeFilter === "uploaded" ? maxSelections === 1 ? "Select Upload" diff --git a/frontend/libs/components/gallery-modal/src/lib/galleryDnd.ts b/frontend/libs/components/gallery-modal/src/lib/galleryDnd.ts index e47fe242df..18f81df6f2 100644 --- a/frontend/libs/components/gallery-modal/src/lib/galleryDnd.ts +++ b/frontend/libs/components/gallery-modal/src/lib/galleryDnd.ts @@ -284,7 +284,9 @@ function onPointerUp(event: PointerEvent) { } else if ( dragState.modalHidden && (dragState.item.mediaClass === "image" || - dragState.item.mediaClass === "dimensional") + dragState.item.mediaClass === "dimensional" || + dragState.item.mediaClass === "mesh" || + dragState.item.mediaClass === "splat") ) { // The cursor left the modal and dropped onto the canvas (not a folder) — // add to the scene. Gated on `modalHidden` so a missed folder drop *inside* diff --git a/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx b/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx index 089a0f8007..2b91842df5 100644 --- a/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx +++ b/frontend/libs/components/pagescene/src/lib/Stage3DBody.tsx @@ -404,17 +404,35 @@ export const Stage3DBody = ({ position.y, ); try { - if (item.mediaClass === "dimensional") { + if ( + item.mediaClass === "dimensional" || + item.mediaClass === "mesh" || + item.mediaClass === "splat" + ) { const isCharacter = item.assetType === "character"; + const isSplat = item.mediaClass === "splat"; const mediaItem: MediaItem = { version: 1, - type: isCharacter ? AssetType.CHARACTER : AssetType.OBJECT, + type: isCharacter + ? AssetType.CHARACTER + : isSplat + ? AssetType.SPLAT + : AssetType.OBJECT, media_id: item.id || uuidv4(), - name: item.label || (isCharacter ? "Character" : "3D Object"), + name: + item.label || + (isCharacter + ? "Character" + : isSplat + ? "3D World" + : "3D Object"), }; if (isCharacter) { await addCharacter(editor, mediaItem, worldPosition); } else { + // NB: Splats intentionally route through addObject too — the + // engine's sceneManager resolves the media token to a splat + // (same as the engine's own DnD in DndAsset.ts). await addObject(editor, mediaItem, worldPosition); } } else {