3d world page and separate store (#932)

frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx

@@ -1,13 +1,12 @@
 import { useEffect, useMemo, useRef, useState } from "react";
 import { animated, useSpring } from "@react-spring/web";
-import { Button, ToggleButton } from "@storyteller/ui-button";
+import { Button } from "@storyteller/ui-button";
 import { TabSelector } from "@storyteller/ui-tab-selector";
 import { Tooltip } from "@storyteller/ui-tooltip";
 import { Viewer3D } from "@storyteller/ui-viewer-3d";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import {
   faCube,
-  faCubes,
   faImages,
   faPlus,
   faSparkles,
@@ -16,11 +15,14 @@ import {
 } from "@fortawesome/pro-solid-svg-icons";
 import { twMerge } from "tailwind-merge";
 import { useImageTo3DStore } from "../../pages/PageImageTo3DObject/ImageTo3DStore";
+import { useImageTo3DWorldStore } from "../../pages/PageImageTo3DWorld/ImageTo3DWorldStore";
 import { MediaUploadApi, downloadFileFromUrl } from "@storyteller/api";
 import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
 import {
   EnqueueImageTo3dObject,
   EnqueueImageTo3dObjectModel,
+  EnqueueImageTo3dWorld,
+  EnqueueImageTo3dWorldModel,
 } from "@storyteller/tauri-api";
 import { toast } from "react-hot-toast";
 import { v4 as uuidv4 } from "uuid";
@@ -68,7 +70,6 @@ export const ImageTo3DExperience = ({
   const [isGenerating, setIsGenerating] = useState(false);
   const [isUploading, setIsUploading] = useState(false);
   const [dragActive, setDragActive] = useState(false);
-  const [meshOnly, setMeshOnly] = useState(false);
   const fileInputRef = useRef<HTMLInputElement>(null);
   const textareaRef = useRef<HTMLTextAreaElement>(null);
 
@@ -79,9 +80,16 @@ export const ImageTo3DExperience = ({
     typeof window !== "undefined" ? window.innerHeight : 800,
   );
 
-  const results = useImageTo3DStore((s) => s.results);
-  const startGeneration = useImageTo3DStore((s) => s.startGeneration);
-  const resetResults = useImageTo3DStore((s) => s.reset);
+  const objectResults = useImageTo3DStore((s) => s.results);
+  const objectStartGeneration = useImageTo3DStore((s) => s.startGeneration);
+  const objectReset = useImageTo3DStore((s) => s.reset);
+
+  const worldResults = useImageTo3DWorldStore((s) => s.results);
+  const worldStartGeneration = useImageTo3DWorldStore((s) => s.startGeneration);
+  const worldReset = useImageTo3DWorldStore((s) => s.reset);
+
+  const results = variant === "object" ? objectResults : worldResults;
+  const resetResults = variant === "object" ? objectReset : worldReset;
   const [uploadedMediaToken, setUploadedMediaToken] = useState<string | null>(
     null,
   );
@@ -106,7 +114,6 @@ export const ImageTo3DExperience = ({
     return () => ro.disconnect();
   }, []);
 
-
   useEffect(() => {
     if (textareaRef.current) {
       textareaRef.current.style.height = "auto";
@@ -114,7 +121,6 @@ export const ImageTo3DExperience = ({
     }
   });
 
-
   const handleFiles = async (files?: FileList | null) => {
     if (!files || files.length === 0) return;
     const file = files[0];
@@ -212,21 +218,33 @@ export const ImageTo3DExperience = ({
           ? snapshotPrompt
           : snapshotName || "Generated Model";
 
-      startGeneration(
-        activeMode,
-        note,
-        snapshotPreview,
-        meshOnly,
-        subscriberId,
-      );
+      if (variant === "object") {
+        objectStartGeneration(
+          activeMode,
+          note,
+          snapshotPreview,
+          false,
+          subscriberId,
+        );
+      } else {
+        worldStartGeneration(activeMode, note, snapshotPreview, subscriberId);
+      }
       setSelectedResultId(subscriberId);
 
-      const result = await EnqueueImageTo3dObject({
-        image_media_token: uploadedMediaToken || undefined,
-        model: EnqueueImageTo3dObjectModel.Hunyuan3d2,
-        frontend_caller: "mini_app",
-        frontend_subscriber_id: subscriberId,
-      });
+      const result =
+        variant === "object"
+          ? await EnqueueImageTo3dObject({
+              image_media_token: uploadedMediaToken || undefined,
+              model: EnqueueImageTo3dObjectModel.Hunyuan3d2,
+              frontend_caller: "mini_app",
+              frontend_subscriber_id: subscriberId,
+            })
+          : await EnqueueImageTo3dWorld({
+              image_media_token: uploadedMediaToken || undefined,
+              model: EnqueueImageTo3dWorldModel.Hunyuan3d2,
+              frontend_caller: "mini_app",
+              frontend_subscriber_id: subscriberId,
+            });
 
       if ("error_type" in result) {
         throw new Error(result.error_message || result.error_type);
@@ -427,7 +445,7 @@ export const ImageTo3DExperience = ({
   }, [activeResult]);
 
   return (
-    <div className="flex h-[calc(100vh-56px)] w-full bg-ui-panel-gradient bg-ui-panel text-base-fg">
+    <div className="bg-ui-panel-gradient flex h-[calc(100vh-56px)] w-full bg-ui-panel text-base-fg">
       {backgroundImage && !hasResults && (
         <>
           <div className="pointer-events-none fixed inset-0 z-[1] overflow-hidden bg-[radial-gradient(50%_50%_at_50%_50%,_transparent_49%,_rgb(var(--st-controls-rgb)_/_var(--st-gallery-vignette-alpha))_100%)]" />
@@ -457,7 +475,10 @@ export const ImageTo3DExperience = ({
         {/* Split View: Viewer + History */}
         {hasResults && (
           <div
-            className="mx-auto grid h-full w-full max-w-7xl grid-cols-[1fr_300px] gap-4 overflow-hidden pb-10"
+            className={twMerge(
+              "mx-auto grid h-full w-full grid-cols-[1fr_300px] gap-4 overflow-hidden pb-10",
+              variant === "world" ? "max-w-[1600px]" : "max-w-7xl",
+            )}
             style={{ height: `calc(100vh - ${bottomOffsetPx + 80}px)` }}
           >
             {/* Left: Viewer */}
@@ -483,7 +504,9 @@ export const ImageTo3DExperience = ({
                           media_id: activeResult.mediaToken!,
                           name: activeResult.note || "3D Object",
                           position: { x: 0, y: 0, z: 0 },
-                        } as MediaItem & { position: { x: number; y: number; z: number } };
+                        } as MediaItem & {
+                          position: { x: number; y: number; z: number };
+                        };
                         addObject(mediaItem);
                         toast.success("Added to 3D scene");
                       }, 500);
@@ -495,11 +518,14 @@ export const ImageTo3DExperience = ({
                     variant="action"
                     icon={faCube}
                     onClick={() => {
-                      toast.promise(downloadFileFromUrl(activeResult.modelUrl!), {
-                        loading: "Downloading GLB...",
-                        success: "Downloaded GLB file",
-                        error: "Failed to download file",
-                      });
+                      toast.promise(
+                        downloadFileFromUrl(activeResult.modelUrl!),
+                        {
+                          loading: "Downloading GLB...",
+                          success: "Downloaded GLB file",
+                          error: "Failed to download file",
+                        },
+                      );
                     }}
                   >
                     GLB
@@ -579,25 +605,16 @@ export const ImageTo3DExperience = ({
             <div className="glass w-full rounded-xl p-5 shadow-2xl ring-1 ring-white/10">
               <div className="space-y-5">{renderActiveMode()}</div>
 
-              <div className="mt-6 flex flex-wrap items-center justify-between gap-3">
-                <ToggleButton
-                  isActive={meshOnly}
-                  icon={faCubes}
-                  activeIcon={faCubes}
-                  label="Mesh only"
-                  onClick={() => setMeshOnly((prev) => !prev)}
-                />
-                <div className="flex items-center gap-3">
-                  <Button
-                    variant="primary"
-                    icon={faSparkles}
-                    disabled={!canGenerate}
-                    onClick={handleGenerate}
-                    loading={isGenerating}
-                  >
-                    {`Generate ${variant === "object" ? "Object" : "World"}`}
-                  </Button>
-                </div>
+              <div className="mt-6 flex justify-center">
+                <Button
+                  variant="primary"
+                  icon={faSparkles}
+                  disabled={!canGenerate}
+                  onClick={handleGenerate}
+                  loading={isGenerating}
+                >
+                  {`Generate ${variant === "object" ? "Object" : "World"}`}
+                </Button>
               </div>
             </div>
 

frontend/apps/editor3d/app/src/config/appMenu.tsx

@@ -117,9 +117,9 @@ export const ALL_APPS: FullAppItem[] = [
     description: "Turn mood boards into explorable worlds",
     icon: faGlobe,
     category: "generate",
-    // action: "IMAGE_TO_3D_WORLD",
+    action: "IMAGE_TO_3D_WORLD",
     color: "bg-blue-500/40",
-    badge: "SOON",
+    badge: "NEW",
   },
   {
     id: "edit-image",

frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorld.tsx

@@ -4,12 +4,11 @@ export const ImageTo3DWorld = () => {
   return (
     <ImageTo3DExperience
       title="Image to 3D World"
-      subtitle="Fuse mood boards, paint-overs, and prompts into explorable scenes with lighting suggestions."
+      subtitle="Generate 3D worlds from an image with World Labs"
       variant="world"
+      backgroundImage="/resources/images/room-of-items.png"
     />
   );
 };
 
 export default ImageTo3DWorld;
-
-

frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/ImageTo3DWorldStore.ts

@@ -0,0 +1,224 @@
+import { create } from "zustand";
+import { listen } from "@tauri-apps/api/event";
+import { MediaFilesApi, MediaUploadApi } from "@storyteller/api";
+import { v4 as uuidv4 } from "uuid";
+
+export type ImageTo3DWorldResult = {
+  id: string;
+  mode: "image" | "text";
+  timestamp: number;
+  note?: string;
+  previewUrl?: string;
+  status: "pending" | "completed";
+  subscriberId: string;
+  modelUrl?: string;
+  mediaToken?: string;
+  coverImageUploaded?: boolean;
+};
+
+export const worldCoverImageCache = new Map<string, string>();
+
+type ImageTo3DWorldState = {
+  results: ImageTo3DWorldResult[];
+  startGeneration: (
+    mode: "image" | "text",
+    note: string,
+    previewUrl: string | undefined,
+    subscriberId?: string,
+  ) => string;
+  completeGeneration: (
+    modelUrl: string,
+    mediaToken: string,
+    maybeSubscriberId?: string,
+  ) => void;
+  uploadCoverFromPreview: (mediaToken: string) => Promise<void>;
+  reset: () => void;
+};
+
+export const useImageTo3DWorldStore = create<ImageTo3DWorldState>(
+  (set, get) => ({
+    results: [],
+    startGeneration: (
+      mode: "image" | "text",
+      note: string,
+      previewUrl: string | undefined,
+      subscriberId?: string,
+    ) => {
+      const id = subscriberId
+        ? subscriberId
+        : crypto.randomUUID
+          ? crypto.randomUUID()
+          : Math.random().toString(36).slice(2);
+      const result: ImageTo3DWorldResult = {
+        id,
+        mode,
+        timestamp: Date.now(),
+        note,
+        previewUrl,
+        status: "pending",
+        subscriberId: id,
+      };
+      set((s) => ({ results: [result, ...s.results] }));
+      return id;
+    },
+    completeGeneration: (
+      modelUrl: string,
+      mediaToken: string,
+      maybeSubscriberId?: string,
+    ) => {
+      console.log("[ImageTo3DWorldStore] completeGeneration", {
+        modelUrl,
+        mediaToken,
+        maybeSubscriberId,
+      });
+      const pending = maybeSubscriberId
+        ? get().results.find((r) => r.subscriberId === maybeSubscriberId)
+        : get().results.find((r) => r.status === "pending");
+
+      set((s) => {
+        const results = [...s.results];
+        const targetIdx = pending
+          ? results.findIndex((r) => r.subscriberId === pending.subscriberId)
+          : -1;
+
+        if (targetIdx === -1) {
+          const generatedId =
+            crypto.randomUUID?.() ?? Math.random().toString(36).slice(2);
+          const result: ImageTo3DWorldResult = {
+            id: generatedId,
+            subscriberId: generatedId,
+            mode: "image",
+            timestamp: Date.now(),
+            note: "Generated World",
+            status: "completed",
+            modelUrl,
+            mediaToken,
+          };
+          return { results: [result, ...results] };
+        }
+
+        results[targetIdx] = {
+          ...results[targetIdx],
+          status: "completed",
+          modelUrl,
+          mediaToken,
+        };
+
+        return { results };
+      });
+    },
+    uploadCoverFromPreview: async (mediaToken: string) => {
+      try {
+        const result = get().results.find((r) => r.mediaToken === mediaToken);
+        if (!result) return;
+        if (result.coverImageUploaded) {
+          console.log(
+            "[ImageTo3DWorldStore] Cover image already uploaded for:",
+            mediaToken,
+          );
+          return;
+        }
+
+        if (!result.previewUrl) {
+          console.log(
+            "[ImageTo3DWorldStore] No preview URL available for cover",
+          );
+          return;
+        }
+
+        console.log(
+          "[ImageTo3DWorldStore] Uploading cover from preview for:",
+          mediaToken,
+        );
+
+        const response = await fetch(result.previewUrl);
+        const blob = await response.blob();
+
+        const mediaUploadApi = new MediaUploadApi();
+        const uploadResult = await mediaUploadApi.UploadImage({
+          blob,
+          fileName: `cover-${mediaToken}.png`,
+          uuid: uuidv4(),
+        });
+
+        if (!uploadResult.success || !uploadResult.data) {
+          console.error("[ImageTo3DWorldStore] Failed to upload cover image");
+          return;
+        }
+
+        const coverImageToken = uploadResult.data;
+        console.log(
+          "[ImageTo3DWorldStore] Cover image uploaded:",
+          coverImageToken,
+        );
+
+        const mediaFilesApi = new MediaFilesApi();
+        const setCoverResult = await mediaFilesApi.UpdateCoverImage({
+          mediaFileToken: mediaToken,
+          imageToken: coverImageToken,
+        });
+
+        if (setCoverResult.success) {
+          console.log("[ImageTo3DWorldStore] Cover image set successfully");
+
+          worldCoverImageCache.set(mediaToken, result.previewUrl);
+
+          set((s) => ({
+            results: s.results.map((r) =>
+              r.mediaToken === mediaToken
+                ? { ...r, coverImageUploaded: true }
+                : r,
+            ),
+          }));
+          window.dispatchEvent(
+            new CustomEvent("cover-image-uploaded", {
+              detail: { mediaToken, thumbnailDataUrl: result.previewUrl },
+            }),
+          );
+        } else {
+          console.error(
+            "[ImageTo3DWorldStore] Failed to set cover image:",
+            setCoverResult.errorMessage,
+          );
+        }
+      } catch (error) {
+        console.error(
+          "[ImageTo3DWorldStore] Error uploading cover image:",
+          error,
+        );
+      }
+    },
+    reset: () => set({ results: [] }),
+  }),
+);
+
+interface WorldGenerationEvent {
+  data: {
+    generated_world?: {
+      cdn_url: string;
+      media_token: string;
+    };
+    maybe_frontend_subscriber_id?: string;
+  };
+}
+
+listen<WorldGenerationEvent>("world_generation_complete_event", (event) => {
+  const payload = event.payload?.data;
+  if (payload?.maybe_frontend_subscriber_id && payload?.generated_world) {
+    console.log(
+      "[ImageTo3DWorldStore] Global event received for subscriber:",
+      payload.maybe_frontend_subscriber_id,
+    );
+    useImageTo3DWorldStore
+      .getState()
+      .completeGeneration(
+        payload.generated_world.cdn_url,
+        payload.generated_world.media_token,
+        payload.maybe_frontend_subscriber_id,
+      );
+
+    useImageTo3DWorldStore
+      .getState()
+      .uploadCoverFromPreview(payload.generated_world.media_token);
+  }
+});

frontend/apps/editor3d/app/src/pages/PageImageTo3DWorld/index.ts

@@ -1,3 +1,2 @@
 export * from "./ImageTo3DWorld";
-
-
+export * from "./ImageTo3DWorldStore";

frontend/libs/tauri-api/src/index.ts

@@ -6,6 +6,7 @@ export * from "./lib/enqueue/EnqueueEditImage.js";
 export * from "./lib/enqueue/EnqueueImageBgRemovalCommand.js";
 export * from "./lib/enqueue/EnqueueImageInpaint.js";
 export * from "./lib/enqueue/EnqueueImageTo3dObject.js";
+export * from "./lib/enqueue/EnqueueImageTo3dWorld.js";
 export * from "./lib/enqueue/EnqueueImageToVideo.js";
 export * from "./lib/enqueue/EnqueueTextToImage.js";
 export * from "./lib/events/BasicEventWrapper.js";
@@ -33,4 +34,4 @@ export * from "./lib/sora/SoraImageRemix.js";
 export * from "./lib/sora/useSoraLoginListener.js";
 export * from "./lib/sora/waitForSoraLogin.js";
 export * from "./lib/system/GetAppInfo.js";
-export * from "./lib/util/LoadWithoutCors.js";
+export * from "./lib/util/LoadWithoutCors.js";

frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageTo3dWorld.ts

@@ -0,0 +1,69 @@
+import { invoke } from "@tauri-apps/api/core";
+import { CommandResult } from "../common/CommandStatus";
+
+export enum EnqueueImageTo3dWorldErrorType {
+  ModelNotSpecified = "model_not_specified",
+  ServerError = "server_error",
+  NeedsFalApiKey = "needs_fal_api_key",
+  FalError = "fal_error",
+  NeedsStorytellerCredentials = "needs_storyteller_credentials",
+}
+
+export interface EnqueueImageTo3dWorldRequest {
+  image_media_token?: string;
+  model?: EnqueueImageTo3dWorldModel;
+  frontend_caller?: string;
+  frontend_subscriber_id?: string;
+}
+
+interface EnqueueImageTo3dWorldRequestRaw {
+  image_media_token?: string;
+  model?: EnqueueImageTo3dWorldModel;
+  frontend_caller?: string;
+  frontend_subscriber_id?: string;
+}
+
+export enum EnqueueImageTo3dWorldModel {
+  Hunyuan3d2 = "hunyuan_3d_2",
+  Hunyuan3d2_0 = "hunyuan_3d_2_0",
+  Hunyuan3d2_1 = "hunyuan_3d_2_1",
+}
+
+export interface EnqueueImageTo3dWorldError extends CommandResult {
+  error_type: EnqueueImageTo3dWorldErrorType;
+  error_message?: string;
+}
+
+export interface EnqueueImageTo3dWorldPayload {}
+
+export interface EnqueueImageTo3dWorldSuccess extends CommandResult {
+  payload: EnqueueImageTo3dWorldPayload;
+}
+
+export type EnqueueImageTo3dWorldResult =
+  | EnqueueImageTo3dWorldSuccess
+  | EnqueueImageTo3dWorldError;
+
+export const EnqueueImageTo3dWorld = async (
+  request: EnqueueImageTo3dWorldRequest
+): Promise<EnqueueImageTo3dWorldResult> => {
+  let mutableRequest: EnqueueImageTo3dWorldRequestRaw = {
+    image_media_token: request.image_media_token,
+    model: request.model,
+  };
+
+  if (!!request.frontend_caller) {
+    mutableRequest.frontend_caller = request.frontend_caller;
+  }
+
+  if (!!request.frontend_subscriber_id) {
+    mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id;
+  }
+
+  let result = await invoke("enqueue_image_to_3d_world_command", {
+    request: mutableRequest,
+  });
+
+  return result as EnqueueImageTo3dWorldResult;
+};
+