diff --git a/frontend/apps/editor3d/app/public/resources/images/room-of-items.png b/frontend/apps/editor3d/app/public/resources/images/room-of-items.png new file mode 100644 index 0000000000..b170bfdf96 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/images/room-of-items.png differ diff --git a/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx b/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx index f213f23d65..cccfb8cd24 100644 --- a/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx +++ b/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx @@ -1,13 +1,12 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { animated, useSpring } from "@react-spring/web"; -import { Button, ToggleButton } from "@storyteller/ui-button"; +import { Button } from "@storyteller/ui-button"; import { TabSelector } from "@storyteller/ui-tab-selector"; import { Tooltip } from "@storyteller/ui-tooltip"; import { Viewer3D } from "@storyteller/ui-viewer-3d"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faCube, - faCubes, faImages, faPlus, faSparkles, @@ -16,11 +15,14 @@ import { } from "@fortawesome/pro-solid-svg-icons"; import { twMerge } from "tailwind-merge"; import { useImageTo3DStore } from "../../pages/PageImageTo3DObject/ImageTo3DStore"; +import { useImageTo3DWorldStore } from "../../pages/PageImageTo3DWorld/ImageTo3DWorldStore"; import { MediaUploadApi, downloadFileFromUrl } from "@storyteller/api"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; import { EnqueueImageTo3dObject, EnqueueImageTo3dObjectModel, + EnqueueImageTo3dWorld, + EnqueueImageTo3dWorldModel, } from "@storyteller/tauri-api"; import { toast } from "react-hot-toast"; import { v4 as uuidv4 } from "uuid"; @@ -68,7 +70,6 @@ export const ImageTo3DExperience = ({ const [isGenerating, setIsGenerating] = useState(false); const [isUploading, setIsUploading] = useState(false); const [dragActive, setDragActive] = useState(false); - const [meshOnly, setMeshOnly] = useState(false); const fileInputRef = useRef(null); const textareaRef = useRef(null); @@ -79,9 +80,16 @@ export const ImageTo3DExperience = ({ typeof window !== "undefined" ? window.innerHeight : 800, ); - const results = useImageTo3DStore((s) => s.results); - const startGeneration = useImageTo3DStore((s) => s.startGeneration); - const resetResults = useImageTo3DStore((s) => s.reset); + const objectResults = useImageTo3DStore((s) => s.results); + const objectStartGeneration = useImageTo3DStore((s) => s.startGeneration); + const objectReset = useImageTo3DStore((s) => s.reset); + + const worldResults = useImageTo3DWorldStore((s) => s.results); + const worldStartGeneration = useImageTo3DWorldStore((s) => s.startGeneration); + const worldReset = useImageTo3DWorldStore((s) => s.reset); + + const results = variant === "object" ? objectResults : worldResults; + const resetResults = variant === "object" ? objectReset : worldReset; const [uploadedMediaToken, setUploadedMediaToken] = useState( null, ); @@ -106,7 +114,6 @@ export const ImageTo3DExperience = ({ return () => ro.disconnect(); }, []); - useEffect(() => { if (textareaRef.current) { textareaRef.current.style.height = "auto"; @@ -114,7 +121,6 @@ export const ImageTo3DExperience = ({ } }); - const handleFiles = async (files?: FileList | null) => { if (!files || files.length === 0) return; const file = files[0]; @@ -212,21 +218,33 @@ export const ImageTo3DExperience = ({ ? snapshotPrompt : snapshotName || "Generated Model"; - startGeneration( - activeMode, - note, - snapshotPreview, - meshOnly, - subscriberId, - ); + if (variant === "object") { + objectStartGeneration( + activeMode, + note, + snapshotPreview, + false, + subscriberId, + ); + } else { + worldStartGeneration(activeMode, note, snapshotPreview, subscriberId); + } setSelectedResultId(subscriberId); - const result = await EnqueueImageTo3dObject({ - image_media_token: uploadedMediaToken || undefined, - model: EnqueueImageTo3dObjectModel.Hunyuan3d2, - frontend_caller: "mini_app", - frontend_subscriber_id: subscriberId, - }); + const result = + variant === "object" + ? await EnqueueImageTo3dObject({ + image_media_token: uploadedMediaToken || undefined, + model: EnqueueImageTo3dObjectModel.Hunyuan3d2, + frontend_caller: "mini_app", + frontend_subscriber_id: subscriberId, + }) + : await EnqueueImageTo3dWorld({ + image_media_token: uploadedMediaToken || undefined, + model: EnqueueImageTo3dWorldModel.Hunyuan3d2, + frontend_caller: "mini_app", + frontend_subscriber_id: subscriberId, + }); if ("error_type" in result) { throw new Error(result.error_message || result.error_type); @@ -427,7 +445,7 @@ export const ImageTo3DExperience = ({ }, [activeResult]); return ( -
+
{backgroundImage && !hasResults && ( <>
@@ -457,7 +475,10 @@ export const ImageTo3DExperience = ({ {/* Split View: Viewer + History */} {hasResults && (
{/* Left: Viewer */} @@ -483,7 +504,9 @@ export const ImageTo3DExperience = ({ media_id: activeResult.mediaToken!, name: activeResult.note || "3D Object", position: { x: 0, y: 0, z: 0 }, - } as MediaItem & { position: { x: number; y: number; z: number } }; + } as MediaItem & { + position: { x: number; y: number; z: number }; + }; addObject(mediaItem); toast.success("Added to 3D scene"); }, 500); @@ -495,11 +518,14 @@ export const ImageTo3DExperience = ({ variant="action" icon={faCube} onClick={() => { - toast.promise(downloadFileFromUrl(activeResult.modelUrl!), { - loading: "Downloading GLB...", - success: "Downloaded GLB file", - error: "Failed to download file", - }); + toast.promise( + downloadFileFromUrl(activeResult.modelUrl!), + { + loading: "Downloading GLB...", + success: "Downloaded GLB file", + error: "Failed to download file", + }, + ); }} > GLB @@ -579,25 +605,16 @@ export const ImageTo3DExperience = ({
{renderActiveMode()}
-
- setMeshOnly((prev) => !prev)} - /> -
- -
+
+
diff --git a/frontend/apps/editor3d/app/src/config/appMenu.tsx b/frontend/apps/editor3d/app/src/config/appMenu.tsx index 63ef5c97a5..e117432861 100644 --- a/frontend/apps/editor3d/app/src/config/appMenu.tsx +++ b/frontend/apps/editor3d/app/src/config/appMenu.tsx @@ -117,9 +117,9 @@ export const ALL_APPS: FullAppItem[] = [ description: "Turn mood boards into explorable worlds", icon: faGlobe, category: "generate", - // action: "IMAGE_TO_3D_WORLD", + action: "IMAGE_TO_3D_WORLD", color: "bg-blue-500/40", - badge: "SOON", + badge: "NEW", }, { id: "edit-image", diff --git a/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorld.tsx b/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorld.tsx index 851bd57774..4a026af3f6 100644 --- a/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorld.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorld.tsx @@ -4,12 +4,11 @@ export const ImageTo3DWorld = () => { return ( ); }; export default ImageTo3DWorld; - - diff --git a/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorldStore.ts b/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorldStore.ts new file mode 100644 index 0000000000..4197a51c92 --- /dev/null +++ b/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorldStore.ts @@ -0,0 +1,224 @@ +import { create } from "zustand"; +import { listen } from "@tauri-apps/api/event"; +import { MediaFilesApi, MediaUploadApi } from "@storyteller/api"; +import { v4 as uuidv4 } from "uuid"; + +export type ImageTo3DWorldResult = { + id: string; + mode: "image" | "text"; + timestamp: number; + note?: string; + previewUrl?: string; + status: "pending" | "completed"; + subscriberId: string; + modelUrl?: string; + mediaToken?: string; + coverImageUploaded?: boolean; +}; + +export const worldCoverImageCache = new Map(); + +type ImageTo3DWorldState = { + results: ImageTo3DWorldResult[]; + startGeneration: ( + mode: "image" | "text", + note: string, + previewUrl: string | undefined, + subscriberId?: string, + ) => string; + completeGeneration: ( + modelUrl: string, + mediaToken: string, + maybeSubscriberId?: string, + ) => void; + uploadCoverFromPreview: (mediaToken: string) => Promise; + reset: () => void; +}; + +export const useImageTo3DWorldStore = create( + (set, get) => ({ + results: [], + startGeneration: ( + mode: "image" | "text", + note: string, + previewUrl: string | undefined, + subscriberId?: string, + ) => { + const id = subscriberId + ? subscriberId + : crypto.randomUUID + ? crypto.randomUUID() + : Math.random().toString(36).slice(2); + const result: ImageTo3DWorldResult = { + id, + mode, + timestamp: Date.now(), + note, + previewUrl, + status: "pending", + subscriberId: id, + }; + set((s) => ({ results: [result, ...s.results] })); + return id; + }, + completeGeneration: ( + modelUrl: string, + mediaToken: string, + maybeSubscriberId?: string, + ) => { + console.log("[ImageTo3DWorldStore] completeGeneration", { + modelUrl, + mediaToken, + maybeSubscriberId, + }); + const pending = maybeSubscriberId + ? get().results.find((r) => r.subscriberId === maybeSubscriberId) + : get().results.find((r) => r.status === "pending"); + + set((s) => { + const results = [...s.results]; + const targetIdx = pending + ? results.findIndex((r) => r.subscriberId === pending.subscriberId) + : -1; + + if (targetIdx === -1) { + const generatedId = + crypto.randomUUID?.() ?? Math.random().toString(36).slice(2); + const result: ImageTo3DWorldResult = { + id: generatedId, + subscriberId: generatedId, + mode: "image", + timestamp: Date.now(), + note: "Generated World", + status: "completed", + modelUrl, + mediaToken, + }; + return { results: [result, ...results] }; + } + + results[targetIdx] = { + ...results[targetIdx], + status: "completed", + modelUrl, + mediaToken, + }; + + return { results }; + }); + }, + uploadCoverFromPreview: async (mediaToken: string) => { + try { + const result = get().results.find((r) => r.mediaToken === mediaToken); + if (!result) return; + if (result.coverImageUploaded) { + console.log( + "[ImageTo3DWorldStore] Cover image already uploaded for:", + mediaToken, + ); + return; + } + + if (!result.previewUrl) { + console.log( + "[ImageTo3DWorldStore] No preview URL available for cover", + ); + return; + } + + console.log( + "[ImageTo3DWorldStore] Uploading cover from preview for:", + mediaToken, + ); + + const response = await fetch(result.previewUrl); + const blob = await response.blob(); + + const mediaUploadApi = new MediaUploadApi(); + const uploadResult = await mediaUploadApi.UploadImage({ + blob, + fileName: `cover-${mediaToken}.png`, + uuid: uuidv4(), + }); + + if (!uploadResult.success || !uploadResult.data) { + console.error("[ImageTo3DWorldStore] Failed to upload cover image"); + return; + } + + const coverImageToken = uploadResult.data; + console.log( + "[ImageTo3DWorldStore] Cover image uploaded:", + coverImageToken, + ); + + const mediaFilesApi = new MediaFilesApi(); + const setCoverResult = await mediaFilesApi.UpdateCoverImage({ + mediaFileToken: mediaToken, + imageToken: coverImageToken, + }); + + if (setCoverResult.success) { + console.log("[ImageTo3DWorldStore] Cover image set successfully"); + + worldCoverImageCache.set(mediaToken, result.previewUrl); + + set((s) => ({ + results: s.results.map((r) => + r.mediaToken === mediaToken + ? { ...r, coverImageUploaded: true } + : r, + ), + })); + window.dispatchEvent( + new CustomEvent("cover-image-uploaded", { + detail: { mediaToken, thumbnailDataUrl: result.previewUrl }, + }), + ); + } else { + console.error( + "[ImageTo3DWorldStore] Failed to set cover image:", + setCoverResult.errorMessage, + ); + } + } catch (error) { + console.error( + "[ImageTo3DWorldStore] Error uploading cover image:", + error, + ); + } + }, + reset: () => set({ results: [] }), + }), +); + +interface WorldGenerationEvent { + data: { + generated_world?: { + cdn_url: string; + media_token: string; + }; + maybe_frontend_subscriber_id?: string; + }; +} + +listen("world_generation_complete_event", (event) => { + const payload = event.payload?.data; + if (payload?.maybe_frontend_subscriber_id && payload?.generated_world) { + console.log( + "[ImageTo3DWorldStore] Global event received for subscriber:", + payload.maybe_frontend_subscriber_id, + ); + useImageTo3DWorldStore + .getState() + .completeGeneration( + payload.generated_world.cdn_url, + payload.generated_world.media_token, + payload.maybe_frontend_subscriber_id, + ); + + useImageTo3DWorldStore + .getState() + .uploadCoverFromPreview(payload.generated_world.media_token); + } +}); diff --git a/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/index.ts b/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/index.ts index 587035eed4..f33b424272 100644 --- a/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/index.ts +++ b/frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/index.ts @@ -1,3 +1,2 @@ export * from "./ImageTo3DWorld"; - - +export * from "./ImageTo3DWorldStore"; diff --git a/frontend/libs/tauri-api/src/index.ts b/frontend/libs/tauri-api/src/index.ts index 4218a3ef6b..118445c870 100644 --- a/frontend/libs/tauri-api/src/index.ts +++ b/frontend/libs/tauri-api/src/index.ts @@ -6,6 +6,7 @@ export * from "./lib/enqueue/EnqueueEditImage.js"; export * from "./lib/enqueue/EnqueueImageBgRemovalCommand.js"; export * from "./lib/enqueue/EnqueueImageInpaint.js"; export * from "./lib/enqueue/EnqueueImageTo3dObject.js"; +export * from "./lib/enqueue/EnqueueImageTo3dWorld.js"; export * from "./lib/enqueue/EnqueueImageToVideo.js"; export * from "./lib/enqueue/EnqueueTextToImage.js"; export * from "./lib/events/BasicEventWrapper.js"; @@ -33,4 +34,4 @@ export * from "./lib/sora/SoraImageRemix.js"; export * from "./lib/sora/useSoraLoginListener.js"; export * from "./lib/sora/waitForSoraLogin.js"; export * from "./lib/system/GetAppInfo.js"; -export * from "./lib/util/LoadWithoutCors.js"; \ No newline at end of file +export * from "./lib/util/LoadWithoutCors.js"; diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageTo3dWorld.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageTo3dWorld.ts new file mode 100644 index 0000000000..20f262307c --- /dev/null +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageTo3dWorld.ts @@ -0,0 +1,69 @@ +import { invoke } from "@tauri-apps/api/core"; +import { CommandResult } from "../common/CommandStatus"; + +export enum EnqueueImageTo3dWorldErrorType { + ModelNotSpecified = "model_not_specified", + ServerError = "server_error", + NeedsFalApiKey = "needs_fal_api_key", + FalError = "fal_error", + NeedsStorytellerCredentials = "needs_storyteller_credentials", +} + +export interface EnqueueImageTo3dWorldRequest { + image_media_token?: string; + model?: EnqueueImageTo3dWorldModel; + frontend_caller?: string; + frontend_subscriber_id?: string; +} + +interface EnqueueImageTo3dWorldRequestRaw { + image_media_token?: string; + model?: EnqueueImageTo3dWorldModel; + frontend_caller?: string; + frontend_subscriber_id?: string; +} + +export enum EnqueueImageTo3dWorldModel { + Hunyuan3d2 = "hunyuan_3d_2", + Hunyuan3d2_0 = "hunyuan_3d_2_0", + Hunyuan3d2_1 = "hunyuan_3d_2_1", +} + +export interface EnqueueImageTo3dWorldError extends CommandResult { + error_type: EnqueueImageTo3dWorldErrorType; + error_message?: string; +} + +export interface EnqueueImageTo3dWorldPayload {} + +export interface EnqueueImageTo3dWorldSuccess extends CommandResult { + payload: EnqueueImageTo3dWorldPayload; +} + +export type EnqueueImageTo3dWorldResult = + | EnqueueImageTo3dWorldSuccess + | EnqueueImageTo3dWorldError; + +export const EnqueueImageTo3dWorld = async ( + request: EnqueueImageTo3dWorldRequest +): Promise => { + let mutableRequest: EnqueueImageTo3dWorldRequestRaw = { + image_media_token: request.image_media_token, + model: request.model, + }; + + if (!!request.frontend_caller) { + mutableRequest.frontend_caller = request.frontend_caller; + } + + if (!!request.frontend_subscriber_id) { + mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id; + } + + let result = await invoke("enqueue_image_to_3d_world_command", { + request: mutableRequest, + }); + + return result as EnqueueImageTo3dWorldResult; +}; +