diff --git a/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx b/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx index cccfb8cd24..6626d5d266 100644 --- a/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx +++ b/frontend/apps/editor3d/app/src/components/experiences/ImageTo3DExperience.tsx @@ -22,6 +22,7 @@ import { EnqueueImageTo3dObject, EnqueueImageTo3dObjectModel, EnqueueImageTo3dWorld, + EnqueueImageToGaussian, EnqueueImageTo3dWorldModel, } from "@storyteller/tauri-api"; import { toast } from "react-hot-toast"; @@ -31,6 +32,7 @@ import { addObject } from "../../pages/PageEnigma/signals/objectGroup/addObject" import { set3DPageMounted } from "../../pages/PageEnigma/Editor/editor"; import { AssetType } from "~/enums"; import type { MediaItem } from "../../pages/PageEnigma/models"; +import { GAUSSIAN_MODELS } from "libs/model-list/src/lib/lists/GaussianModels"; type Mode = "image" | "text"; type Variant = "object" | "world"; @@ -206,7 +208,9 @@ export const ImageTo3DExperience = ({ if (activeMode === "image" && !uploadedMediaToken) return; if (activeMode === "text" && prompt.trim().length <= 3) return; + console.log("Set is generating..."); setIsGenerating(true); + const snapshotPrompt = prompt.trim(); const snapshotPreview = uploadedPreview || undefined; const snapshotName = uploadedName; @@ -219,6 +223,7 @@ export const ImageTo3DExperience = ({ : snapshotName || "Generated Model"; if (variant === "object") { + console.log("Start object generation..."); objectStartGeneration( activeMode, note, @@ -227,7 +232,11 @@ export const ImageTo3DExperience = ({ subscriberId, ); } else { + console.log("Start gaussian generation..."); worldStartGeneration(activeMode, note, snapshotPreview, subscriberId); + + + } setSelectedResultId(subscriberId); @@ -239,9 +248,9 @@ export const ImageTo3DExperience = ({ frontend_caller: "mini_app", frontend_subscriber_id: subscriberId, }) - : await EnqueueImageTo3dWorld({ - image_media_token: uploadedMediaToken || undefined, - model: EnqueueImageTo3dWorldModel.Hunyuan3d2, + : await EnqueueImageToGaussian({ + image_media_tokens: uploadedMediaToken ? [uploadedMediaToken] : undefined, + model: GAUSSIAN_MODELS[0], frontend_caller: "mini_app", frontend_subscriber_id: subscriberId, }); diff --git a/frontend/libs/model-list/src/index.ts b/frontend/libs/model-list/src/index.ts index d3fc8c8d1f..18ff980b8d 100644 --- a/frontend/libs/model-list/src/index.ts +++ b/frontend/libs/model-list/src/index.ts @@ -14,4 +14,6 @@ export * from "./lib/lists/AllModels.js"; export * from "./lib/lists/AllModelsById.js"; export * from "./lib/lists/ImageModels.js"; export * from "./lib/lists/VideoModels.js"; +export * from "./lib/lists/GaussianModels.js"; + diff --git a/frontend/libs/model-list/src/lib/classes/GaussianModel.ts b/frontend/libs/model-list/src/lib/classes/GaussianModel.ts new file mode 100644 index 0000000000..faec3c7a2b --- /dev/null +++ b/frontend/libs/model-list/src/lib/classes/GaussianModel.ts @@ -0,0 +1,27 @@ +import { ModelCreator } from "src/index.js"; +import { Model, ModelKind } from "./Model.js"; +import { ModelCategory } from "../legacy/ModelConfig.js"; +import { ModelTag } from "./metadata/ModelTag.js"; + +export class GaussianModel extends Model { + // Typescript type discriminator property + // Since Vite minification and class name mangling can break instanceof checks, + // we have a type discriminator property to check against. + override readonly kind: ModelKind = "gaussian_model"; + + + constructor(args: { + id: string; + tauriId: string; + fullName: string; + category: ModelCategory; + creator: ModelCreator; + selectorName: string; + selectorDescription: string; + selectorBadges: string[]; + tags?: ModelTag[]; + progressBarTime?: number; + }) { + super(args); + } +} diff --git a/frontend/libs/model-list/src/lib/classes/Model.ts b/frontend/libs/model-list/src/lib/classes/Model.ts index d6ee21e428..8ef8190ecd 100644 --- a/frontend/libs/model-list/src/lib/classes/Model.ts +++ b/frontend/libs/model-list/src/lib/classes/Model.ts @@ -2,7 +2,7 @@ import { ModelCreator } from "./metadata/ModelCreator.js"; import { ModelCategory, ModelConfig } from "../legacy/ModelConfig.js"; import { ModelTag } from "./metadata/ModelTag.js"; -export type ModelKind = "model" | "image_model" | "video_model"; +export type ModelKind = "model" | "image_model" | "video_model" | "gaussian_model"; // NB: Do not create instances of this class directly, use subclasses. export class Model { diff --git a/frontend/libs/model-list/src/lib/classes/metadata/ModelCreator.ts b/frontend/libs/model-list/src/lib/classes/metadata/ModelCreator.ts index b80de53c66..c388d4641a 100644 --- a/frontend/libs/model-list/src/lib/classes/metadata/ModelCreator.ts +++ b/frontend/libs/model-list/src/lib/classes/metadata/ModelCreator.ts @@ -20,4 +20,5 @@ export enum ModelCreator { Vidu = "Vidu", ArtCraft = "ArtCraft", Grok = "Grok", + WorldLabs = "WorldLabs", } diff --git a/frontend/libs/model-list/src/lib/legacy/ModelConfig.ts b/frontend/libs/model-list/src/lib/legacy/ModelConfig.ts index 9ff7bf5439..79ab90ea3f 100644 --- a/frontend/libs/model-list/src/lib/legacy/ModelConfig.ts +++ b/frontend/libs/model-list/src/lib/legacy/ModelConfig.ts @@ -1,7 +1,7 @@ import { ModelInfo } from "./ModelInfo.js"; import { ModelTag } from "../classes/metadata/ModelTag.js"; -export type ModelCategory = "image" | "video"; +export type ModelCategory = "image" | "video" | "gaussian"; // Centralized model capability definition // For setting options for the model diff --git a/frontend/libs/model-list/src/lib/lists/AllModels.ts b/frontend/libs/model-list/src/lib/lists/AllModels.ts index fb6ae4b898..b642ab66ef 100644 --- a/frontend/libs/model-list/src/lib/lists/AllModels.ts +++ b/frontend/libs/model-list/src/lib/lists/AllModels.ts @@ -1,5 +1,6 @@ import { IMAGE_MODELS } from "./ImageModels.js"; import { VIDEO_MODELS } from "./VideoModels.js"; import { Model } from "../classes/Model.js"; +import { GAUSSIAN_MODELS } from "./GaussianModels.js"; -export const ALL_MODELS_LIST: Model[] = [...IMAGE_MODELS, ...VIDEO_MODELS]; +export const ALL_MODELS_LIST: Model[] = [...IMAGE_MODELS, ...VIDEO_MODELS, ...GAUSSIAN_MODELS]; diff --git a/frontend/libs/model-list/src/lib/lists/GaussianModels.ts b/frontend/libs/model-list/src/lib/lists/GaussianModels.ts new file mode 100644 index 0000000000..47894ebe08 --- /dev/null +++ b/frontend/libs/model-list/src/lib/lists/GaussianModels.ts @@ -0,0 +1,26 @@ +import { GaussianModel } from "../classes/GaussianModel.js"; +import { ModelCreator } from "../classes/metadata/ModelCreator.js"; + +// TODO: Some of the model configs, such as generation counts, are authoritatively controlled in `legacy/Models.ts` + +export const GAUSSIAN_MODELS : GaussianModel [] = [ + new GaussianModel({ + id: "world_labs_marble", + tauriId: "world_labs_marble", + fullName: "WorldLabs Marble", + category: "gaussian", + creator: ModelCreator.WorldLabs, + selectorName: "WorldLabs Marble", + selectorDescription: "Amazing", + selectorBadges: ["10 sec."], + progressBarTime: 10000, + }), +]; + +export const GAUSSIAN_MODELS_BY_ID: Map = new Map( + GAUSSIAN_MODELS.map((model) => [model.id, model]), +); + +if (GAUSSIAN_MODELS_BY_ID.size !== GAUSSIAN_MODELS.length) { + throw new Error("All gaussian models must have unique IDs"); +} diff --git a/frontend/libs/tauri-api/src/index.ts b/frontend/libs/tauri-api/src/index.ts index 6780b44df2..3cd31253bb 100644 --- a/frontend/libs/tauri-api/src/index.ts +++ b/frontend/libs/tauri-api/src/index.ts @@ -8,6 +8,7 @@ export * from "./lib/enqueue/EnqueueEditImage.js"; export * from "./lib/enqueue/EnqueueImageBgRemovalCommand.js"; export * from "./lib/enqueue/EnqueueImageInpaint.js"; export * from "./lib/enqueue/EnqueueImageTo3dObject.js"; +export * from "./lib/enqueue/EnqueueImageToGaussian.js"; export * from "./lib/enqueue/EnqueueImageTo3dWorld.js"; export * from "./lib/enqueue/EnqueueImageToVideo.js"; export * from "./lib/enqueue/EnqueueTextToImage.js"; diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToGaussian.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToGaussian.ts new file mode 100644 index 0000000000..3645d7b6d0 --- /dev/null +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToGaussian.ts @@ -0,0 +1,81 @@ +import { invoke } from "@tauri-apps/api/core"; +import { CommandResult } from "../common/CommandStatus"; +import { Model } from "@storyteller/model-list"; + +export enum EnqueueImageToGaussianErrorType { + /// Caller didn't specify a model + ModelNotSpecified = "model_not_specified", + /// Generic server error + ServerError = "server_error", + /// No Storyteller creds + NeedsStorytellerCredentials = "needs_storyteller_credentials", + /// No provider available + NoProviderAvailable = "no_provider_available", +} + +export interface EnqueueImageToGaussianRequest { + // Required. The model to use. + model?: Model; + + // Optional. Text prompt to direct the gaussian. + prompt?: string; + + // Required. Starting frame. + image_media_tokens?: string[]; + + // Optional frontend state to return later. + frontend_caller?: string; + + // Optional frontend state to return later. + frontend_subscriber_id?: string; +} + +interface RawEnqueueImageToGaussianRequest { + model?: string; + prompt?: string; + image_media_tokens?: string[]; + frontend_caller?: string; + frontend_subscriber_id?: string; +} + +export interface EnqueueImageToGaussianError extends CommandResult { + error_type: EnqueueImageToGaussianErrorType; + error_message?: string; +} + +export type EnqueueImageToGaussianPayload = Record; + +export interface EnqueueImageToGaussianSuccess extends CommandResult { + payload: EnqueueImageToGaussianPayload; +} + +export type EnqueueImageToGaussianResult = + | EnqueueImageToGaussianSuccess + | EnqueueImageToGaussianError; + +export const EnqueueImageToGaussian = async ( + request: EnqueueImageToGaussianRequest +): Promise => { + const mutableRequest: RawEnqueueImageToGaussianRequest = { + model: request.model?.tauriId, + image_media_tokens: request.image_media_tokens, + }; + + if (request.prompt) { + mutableRequest.prompt = request.prompt; + } + + if (request.frontend_caller) { + mutableRequest.frontend_caller = request.frontend_caller; + } + + if (request.frontend_subscriber_id) { + mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id; + } + + const result = await invoke("enqueue_image_to_gaussian_command", { + request: mutableRequest, + }); + + return result as EnqueueImageToGaussianResult; +};