From 11bf5949feb6d958dbc22ccc2eda773ee7435691 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 28 Dec 2025 09:30:41 -0500 Subject: [PATCH] select provider in frontend (#944) --- .../image_edit/enqueue_edit_image_command.rs | 13 ++++--- .../enqueue_image_inpaint_command.rs | 11 ++++-- .../enqueue_image_to_video_command.rs | 27 +++++++++++--- .../enqueue_text_to_image_command.rs | 2 ++ .../app/src/pages/PageDraw/PageDraw.tsx | 6 ++++ .../PageDraw/PromptEditor/PromptEditor.tsx | 2 ++ .../src/pages/PageDraw/PromptEditor/types.ts | 2 ++ .../app/src/pages/PageEdit/PageEdit.tsx | 36 +++++++++++++++---- .../app/src/pages/PageEnigma/PageEditor.tsx | 6 ++++ .../app/src/pages/PageImage/TextToImage.tsx | 6 ++++ .../app/src/pages/PageVideo/ImageToVideo.tsx | 6 ++++ .../src/lib/common/generation_provider.ts | 5 ++- .../components/model-selector/src/index.ts | 1 - .../src/lib/classy-model-selector-store.ts | 11 +++--- .../src/lib/classy-model-selector.tsx | 17 +++------ .../model-selector/src/lib/provider-icons.tsx | 36 +++++++++---------- .../model-selector/src/lib/provider-lookup.ts | 23 ------------ .../components/model-selector/tsconfig.json | 3 ++ .../model-selector/tsconfig.lib.json | 3 ++ .../promptbox/src/lib/PromptBox2D.tsx | 14 ++++++-- .../promptbox/src/lib/PromptBox3D.tsx | 14 ++++++-- .../promptbox/src/lib/PromptBoxEdit.tsx | 5 +++ .../promptbox/src/lib/PromptBoxImage.tsx | 7 ++++ .../promptbox/src/lib/PromptBoxVideo.tsx | 7 ++++ .../model-list/src/lib/classes/ImageModel.ts | 2 ++ .../libs/model-list/src/lib/classes/Model.ts | 10 ++++++ .../model-list/src/lib/classes/VideoModel.ts | 2 ++ .../model-list/src/lib/lists/ImageModels.ts | 11 ++++++ .../model-list/src/lib/lists/VideoModels.ts | 8 +++++ frontend/libs/model-list/tsconfig.json | 3 ++ frontend/libs/model-list/tsconfig.lib.json | 3 ++ .../src/lib/enqueue/EnqueueEditImage.ts | 10 +++++- .../src/lib/enqueue/EnqueueImageInpaint.ts | 9 +++++ .../src/lib/enqueue/EnqueueImageToVideo.ts | 9 +++++ .../src/lib/enqueue/EnqueueTextToImage.ts | 11 +++++- .../tauri-api/src/lib/providers/Provider.ts | 3 +- 36 files changed, 258 insertions(+), 86 deletions(-) delete mode 100644 frontend/libs/components/model-selector/src/lib/provider-lookup.ts diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs index 042210c385..7e6fd0d75a 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs @@ -277,8 +277,13 @@ pub async fn handle_request( } }; - let mut success_event = match request.provider { - None | Some(GenerationProvider::Artcraft) => { + let provider = request.provider + .unwrap_or(GenerationProvider::Artcraft); + + info!("edit image with {:?} via provider {:?}", &model, &provider); + + let mut success_event = match provider { + GenerationProvider::Artcraft => { handle_image_edit_artcraft( model, request, @@ -288,7 +293,7 @@ pub async fn handle_request( storyteller_creds_manager, ).await? } - Some(GenerationProvider::Sora) => { + GenerationProvider::Sora => { handle_image_edit_sora( model, request, @@ -301,7 +306,7 @@ pub async fn handle_request( } _ => { return Err(GenerateError::BadProviderForModel { - provider: request.provider.unwrap_or(GenerationProvider::Artcraft), + provider, model: image_edit_model_to_model_type(model), }) } diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs index 95263c2943..b731ea208f 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs @@ -234,8 +234,13 @@ pub async fn handle_request( } }; - let success_event = match request.provider { - None | Some(GenerationProvider::Artcraft) => { + let provider = request.provider + .unwrap_or(GenerationProvider::Artcraft); + + info!("image inpaint with {:?} via provider {:?}", &model, &provider); + + let success_event = match provider { + GenerationProvider::Artcraft => { handle_inpaint_artcraft( model, request, @@ -245,7 +250,7 @@ pub async fn handle_request( storyteller_creds_manager, ).await? } - Some(provider) => { + _ => { return Err(GenerateError::BadProviderForModel { provider, model: image_inpaint_model_to_model_type(model), diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs index 1366e77ba1..a279cfbdc5 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs @@ -4,6 +4,7 @@ use crate::core::commands::enqueue::image_to_video::artcraft::handle_artcraft_vi use crate::core::commands::enqueue::image_to_video::grok::handle_grok_video::handle_grok_video; use crate::core::commands::enqueue::image_to_video::sora2::handle_sora_sora2::handle_sora_sora2; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::TextToImageModel; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::Response; use crate::core::commands::response::success_response_wrapper::SerializeMarker; @@ -20,6 +21,7 @@ use crate::services::grok::state::grok_credential_manager::GrokCredentialManager use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use enums::common::generation_provider::GenerationProvider; use enums::tauri::ux::tauri_command_caller::TauriCommandCaller; use log::{error, info, warn}; use serde_derive::{Deserialize, Serialize}; @@ -76,6 +78,10 @@ pub enum VideoModel { #[derive(Deserialize, Debug)] pub struct EnqueueImageToVideoRequest { + /// The provider to use (defaults to Artcraft/Storyteller). + /// Not all (provider, model) combinations are valid. + pub provider: Option, + /// REQUIRED. /// The model to use. pub model: Option, @@ -263,8 +269,22 @@ pub async fn handle_request( storyteller_creds_manager: &StorytellerCredentialManager, ) -> Result { - let result = match request.model { - Some(VideoModel::GrokVideo) => { + let model = match request.model { + Some(model) => model, + None => { + return Err(GenerateError::no_model_specified()) + } + }; + + let provider = match (model, request.provider) { + (VideoModel::GrokVideo, _) => GenerationProvider::Grok, + _ => request.provider.unwrap_or(GenerationProvider::Artcraft), + }; + + info!("generate video with {:?} via provider {:?}", &model, &provider); + + let result = match provider { + GenerationProvider::Grok => { handle_grok_video( &request, app, @@ -273,8 +293,7 @@ pub async fn handle_request( grok_creds_manager, ).await } - Some(VideoModel::Sora2) => { - // TODO: Route based on forthcoming provider parameter. + GenerationProvider::Sora => { handle_sora_sora2( &request, app, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs index d0d5f34ba1..947d607fbc 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs @@ -333,6 +333,8 @@ pub async fn dispatch_request( _ => request.provider.unwrap_or(GenerationProvider::Artcraft), }; + info!("generate image with {:?} via provider {:?}", &model, &provider); + match provider { GenerationProvider::Artcraft => { handle_text_to_image_artcraft( diff --git a/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx b/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx index 3b31d7eedf..09ac612751 100644 --- a/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageDraw/PageDraw.tsx @@ -19,9 +19,11 @@ import { //ProviderSelector, //PROVIDER_LOOKUP_BY_PAGE, useSelectedImageModel, + useSelectedProviderForModel, } from "@storyteller/ui-model-selector"; import { useCanvasBgRemovedEvent } from "@storyteller/tauri-api"; import { TutorialModalButton } from "@storyteller/ui-tutorial-modal"; +import { GenerationProvider } from "@storyteller/api-enums"; const PAGE_ID: ModelPage = ModelPage.Canvas2D; @@ -61,6 +63,9 @@ const PageDraw = () => { const selectedImageModel: ImageModel | undefined = useSelectedImageModel(PAGE_ID); + const selectedProvider : GenerationProvider | undefined = + useSelectedProviderForModel(PAGE_ID, selectedImageModel?.id); + useDeleteHotkeys({ onDelete: store.deleteSelectedItems }); useUndoRedoHotkeys({ undo: store.undo, redo: store.redo }); useCopyPasteHotkeys({ @@ -304,6 +309,7 @@ const PageDraw = () => { onFitPressed={onFitPressed} //selectedModelInfo={selectedModelInfo} selectedImageModel={selectedImageModel} + selectedProvider={selectedProvider} /> = ({ onFitPressed, //selectedModelInfo, selectedImageModel, + selectedProvider, }) => { const [images, setImages] = useState([]); @@ -48,6 +49,7 @@ const PromptEditor: React.FC = ({ onFitPressed={onFitPressed} //selectedModelInfo={selectedModelInfo} selectedImageModel={selectedImageModel} + selectedProvider={selectedProvider} /> diff --git a/frontend/apps/editor3d/app/src/pages/PageDraw/PromptEditor/types.ts b/frontend/apps/editor3d/app/src/pages/PageDraw/PromptEditor/types.ts index 35a32f679c..1192dfaf2a 100644 --- a/frontend/apps/editor3d/app/src/pages/PageDraw/PromptEditor/types.ts +++ b/frontend/apps/editor3d/app/src/pages/PageDraw/PromptEditor/types.ts @@ -1,5 +1,6 @@ import { ImageModel } from "@storyteller/model-list"; import { ModelInfo } from "@storyteller/model-list"; +import { GenerationProvider } from "@storyteller/api-enums"; export interface PromptEditorProps { initialPrompt?: string; @@ -13,6 +14,7 @@ export interface PromptEditorProps { onFitPressed?: () => void; selectedModelInfo?: ModelInfo; selectedImageModel?: ImageModel; + selectedProvider?: GenerationProvider; } export type AspectRatio = "1:1" | "3:2" | "2:3"; diff --git a/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx b/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx index c8f10f6dd8..57f8aa22f6 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx @@ -5,6 +5,8 @@ import { EnqueueImageInpaint, EnqueueEditImageSize, EnqueueEditImageResolution, + EnqueueEditImageRequest, + EnqueueImageInpaintRequest, } from "@storyteller/tauri-api"; import { PromptsApi } from "@storyteller/api"; import { RefImage, usePromptEditStore } from "@storyteller/ui-promptbox"; @@ -21,6 +23,7 @@ import MarkerToolControlBar from "./MarkerToolControlBar"; import { ClassyModelSelector, useSelectedImageModel, + useSelectedProviderForModel, IMAGE_EDITOR_PAGE_MODEL_LIST, ModelPage, //ProviderSelector, @@ -29,6 +32,8 @@ import { import { ImageModel } from "@storyteller/model-list"; import { HistoryStack, ImageBundle } from "./HistoryStack"; import { TutorialModalButton } from "@storyteller/ui-tutorial-modal"; +import { GenerationProvider } from "@storyteller/api-enums"; + const PAGE_ID: ModelPage = ModelPage.ImageEditor; @@ -38,6 +43,9 @@ const PageEdit = () => { const selectedImageModel: ImageModel | undefined = useSelectedImageModel(PAGE_ID); + const selectedProvider : GenerationProvider | undefined = + useSelectedProviderForModel(PAGE_ID, selectedImageModel?.id); + // State for canvas dimensions const canvasWidth = useRef(1024); const canvasHeight = useRef(1024); @@ -322,6 +330,7 @@ const PageEdit = () => { aspectRatio?: string; resolution?: string; images?: RefImage[]; + selectedProvider?: GenerationProvider; }, ) => { const editedImageToken = store.baseImageInfo?.mediaToken; @@ -374,7 +383,7 @@ const PageEdit = () => { if (selectedImageModel?.editingIsInpainting) { const arrayBuffer = await getMaskArrayBuffer(); - result = await EnqueueImageInpaint({ + let request : EnqueueImageInpaintRequest = { model: selectedImageModel, image_media_token: editedImageToken, mask_image_raw_bytes: arrayBuffer, @@ -382,7 +391,11 @@ const PageEdit = () => { image_count: generationCount, frontend_caller: "image_editor", frontend_subscriber_id: subscriberId, - }); + }; + if (!!options?.selectedProvider) { + request.provider = options.selectedProvider; + } + result = await EnqueueImageInpaint(request); } else if (selectedImageModel?.isNanoBananaModel()) { const compositeFile = await getCompositeCanvasFile(); if (!compositeFile) { @@ -398,7 +411,7 @@ const PageEdit = () => { return; } const imgs = options?.images || []; - result = await EnqueueEditImage({ + let request: EnqueueEditImageRequest = { model: selectedImageModel, scene_image_media_token: snapshotResult.data, image_media_tokens: imgs @@ -410,10 +423,14 @@ const PageEdit = () => { frontend_subscriber_id: subscriberId, aspect_ratio: mapAspectRatio(options?.aspectRatio), image_resolution: mapResolution(options?.resolution), - }); + }; + if (!!options?.selectedProvider) { + request.provider = options.selectedProvider; + } + result = await EnqueueEditImage(request); } else { const imgs = options?.images || []; - result = await EnqueueEditImage({ + let request : EnqueueEditImageRequest = { model: selectedImageModel, image_media_tokens: [ editedImageToken, @@ -428,10 +445,14 @@ const PageEdit = () => { frontend_subscriber_id: subscriberId, aspect_ratio: mapAspectRatio(options?.aspectRatio), image_resolution: mapResolution(options?.resolution), - }); + }; + if (!!options?.selectedProvider) { + request.provider = options.selectedProvider; + } + result = await EnqueueEditImage(request); } - if (result.status === "success") { + if (result?.status === "success") { setPendingGenerations((prev) => [ ...prev, { id: subscriberId as string, count: generationCount }, @@ -609,6 +630,7 @@ const PageEdit = () => { > { store.setActiveTool(mode as ActiveEditTool); }} diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx index 12b432ff78..746cb3937c 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx @@ -77,6 +77,7 @@ import { STAGE_3D_PAGE_MODEL_LIST, ModelPage, useSelectedImageModel, + useSelectedProviderForModel, ClassyModelSelector, //ProviderSelector, //PROVIDER_LOOKUP_BY_PAGE, @@ -87,6 +88,7 @@ import { useTabStore } from "../Stores/TabState"; import PageEdit from "../PageEdit/PageEdit"; import { ImageModel } from "@storyteller/model-list"; import { TutorialModalButton } from "@storyteller/ui-tutorial-modal"; +import { GenerationProvider } from "@storyteller/api-enums"; const PAGE_ID: ModelPage = ModelPage.Stage3D; @@ -117,6 +119,9 @@ export const PageEditor = () => { const selectedImageModel: ImageModel | undefined = useSelectedImageModel(PAGE_ID); + const selectedProvider : GenerationProvider | undefined = + useSelectedProviderForModel(PAGE_ID, selectedImageModel?.id); + const height = dndTimelineHeight.value > -1 ? pageHeight.value - dndTimelineHeight.value - 56 @@ -530,6 +535,7 @@ export const PageEditor = () => { handleCameraFocalLengthChange={handleCameraFocalLengthChange} onAspectRatioSelect={onAspectRatioSelect} selectedImageModel={selectedImageModel} + selectedProvider={selectedProvider} setEnginePrompt={(prompt) => { console.log("setEnginePrompt", prompt); if (!editorEngine) { diff --git a/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx b/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx index a333c86d28..5043b9ba92 100644 --- a/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageImage/TextToImage.tsx @@ -8,6 +8,7 @@ import { ModelPage, ClassyModelSelector, useSelectedImageModel, + useSelectedProviderForModel, //ProviderSelector, //PROVIDER_LOOKUP_BY_PAGE, } from "@storyteller/ui-model-selector"; @@ -27,6 +28,7 @@ import { import { Badge } from "@storyteller/ui-badge"; import { twMerge } from "tailwind-merge"; import { TutorialModalButton } from "@storyteller/ui-tutorial-modal"; +import { GenerationProvider } from "@storyteller/api-enums"; const PAGE_ID: ModelPage = ModelPage.TextToImage; @@ -43,6 +45,9 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => { const selectedImageModel: ImageModel | undefined = useSelectedImageModel(PAGE_ID); + const selectedProvider : GenerationProvider | undefined = + useSelectedProviderForModel(PAGE_ID, selectedImageModel?.id); + const jobContext: JobContextType = { jobTokens: [], addJobToken: () => {}, @@ -214,6 +219,7 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => { }} uploadImage={uploadImage} selectedModel={selectedImageModel} + selectedProvider={selectedProvider} imageMediaId={imageMediaId} url={imageUrl ?? undefined} onImageRowVisibilityChange={setImageRowVisible} diff --git a/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx b/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx index 9e2705cf48..8eeff7c1fb 100644 --- a/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx @@ -7,6 +7,7 @@ import { IMAGE_TO_VIDEO_PAGE_MODEL_LIST, ModelPage, useSelectedVideoModel, + useSelectedProviderForModel, //ProviderSelector, //PROVIDER_LOOKUP_BY_PAGE, } from "@storyteller/ui-model-selector"; @@ -21,6 +22,7 @@ import { import { twMerge } from "tailwind-merge"; import { uploadImage } from "../../components/reusable/UploadModalMedia/uploadImage"; import { TutorialModalButton } from "@storyteller/ui-tutorial-modal"; +import { GenerationProvider } from "@storyteller/api-enums"; const PAGE_ID: ModelPage = ModelPage.ImageToVideo; @@ -42,6 +44,9 @@ const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => { const selectedVideoModel: VideoModel | undefined = useSelectedVideoModel(PAGE_ID); + const selectedProvider: GenerationProvider | undefined = + useSelectedProviderForModel(PAGE_ID, selectedVideoModel?.id); + const jobContext: JobContextType = { jobTokens: [], addJobToken: () => {}, @@ -186,6 +191,7 @@ const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => { jobContext} selectedModel={selectedVideoModel} + selectedProvider={selectedProvider} imageMediaId={imageMediaId} url={imageUrl ?? undefined} onImageRowVisibilityChange={setImageRowVisible} diff --git a/frontend/libs/api-enums/src/lib/common/generation_provider.ts b/frontend/libs/api-enums/src/lib/common/generation_provider.ts index 6c1c7d3502..cf41aabe62 100644 --- a/frontend/libs/api-enums/src/lib/common/generation_provider.ts +++ b/frontend/libs/api-enums/src/lib/common/generation_provider.ts @@ -4,7 +4,10 @@ export enum GenerationProvider { Artcraft = "artcraft", - Fal = "fal", + Grok = "grok", Midjourney = "midjourney", Sora = "sora", + WorldLabs = "world_labs", + // NB: We should build our own open source Fal + //Fal = "fal", } diff --git a/frontend/libs/components/model-selector/src/index.ts b/frontend/libs/components/model-selector/src/index.ts index 7d5762771b..f1fad278af 100644 --- a/frontend/libs/components/model-selector/src/index.ts +++ b/frontend/libs/components/model-selector/src/index.ts @@ -4,4 +4,3 @@ export * from "./lib/classy-model-selector"; export * from "./lib/classy-model-selector-store"; export * from "./lib/provider-selector"; export * from "./lib/provider-icons"; -export * from "./lib/provider-lookup"; diff --git a/frontend/libs/components/model-selector/src/lib/classy-model-selector-store.ts b/frontend/libs/components/model-selector/src/lib/classy-model-selector-store.ts index 35a2a89636..266eb7d87e 100644 --- a/frontend/libs/components/model-selector/src/lib/classy-model-selector-store.ts +++ b/frontend/libs/components/model-selector/src/lib/classy-model-selector-store.ts @@ -1,16 +1,16 @@ import { create } from "zustand"; import { ModelPage } from "./model-pages"; import { ImageModel, Model, VideoModel } from "@storyteller/model-list"; -import { Provider } from "@storyteller/tauri-api"; +import { GenerationProvider } from "@storyteller/api-enums"; interface ClassyModelSelectorState { selectedModels: { [page in ModelPage]?: Model }; - selectedProviders: { [page in ModelPage]?: { [modelId: string]: Provider } }; + selectedProviders: { [page in ModelPage]?: { [modelId: string]: GenerationProvider } }; setSelectedModel: (page: ModelPage, model: Model) => void; setSelectedProvider: ( page: ModelPage, modelId: string, - provider: Provider + provider: GenerationProvider ) => void; } @@ -73,7 +73,7 @@ export const getSelectedVideoModel = ( export const getSelectedProviderForModel = ( page: ModelPage, modelId: string -): Provider | undefined => { +): GenerationProvider | undefined => { const { selectedProviders } = useClassyModelSelectorStore.getState(); const byPage = selectedProviders[page]; if (!byPage) return undefined; @@ -104,10 +104,11 @@ export const useSelectedVideoModel = ( : undefined; }; +// TODO: This shouldn't be on a per-page basis. export const useSelectedProviderForModel = ( page: ModelPage, modelId: string | undefined -): Provider | undefined => +): GenerationProvider | undefined => useClassyModelSelectorStore((s) => modelId ? s.selectedProviders[page]?.[modelId] : undefined ); diff --git a/frontend/libs/components/model-selector/src/lib/classy-model-selector.tsx b/frontend/libs/components/model-selector/src/lib/classy-model-selector.tsx index 10bbf8538c..e40998b454 100644 --- a/frontend/libs/components/model-selector/src/lib/classy-model-selector.tsx +++ b/frontend/libs/components/model-selector/src/lib/classy-model-selector.tsx @@ -10,7 +10,7 @@ import { getProviderDisplayName, getProviderIcon } from "./provider-icons"; import { Model } from "@storyteller/model-list"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faCircleCheck, faChevronUp } from "@fortawesome/pro-solid-svg-icons"; -import { PROVIDER_LOOKUP_BY_PAGE } from "./provider-lookup"; +import { GenerationProvider } from "@storyteller/api-enums"; interface ClassyModelSelectorProps { items: Omit[]; @@ -26,7 +26,7 @@ interface ClassyModelSelectorProps { maxListHeight?: number | string; } -const DEFAULT_PROVIDER_OPTIONS: Provider[] = [Provider.ArtCraft]; +const DEFAULT_PROVIDER_OPTIONS: GenerationProvider[] = [GenerationProvider.Artcraft]; function ProviderTooltipContent({ page, @@ -40,7 +40,7 @@ function ProviderTooltipContent({ modelId?: string; model?: Model; modelLabel?: string; - allowedProviders: Provider[]; + allowedProviders: GenerationProvider[]; onFinished?: () => void; }) { const { setSelectedModel, setSelectedProvider } = @@ -126,10 +126,7 @@ export function ClassyModelSelector({ const modelId = item.model?.id; if (!modelId) continue; if (selectedProvidersByModel[modelId]) continue; - const allowed = - providersByModel?.[modelId] ?? - PROVIDER_LOOKUP_BY_PAGE[page]?.[modelId] ?? - DEFAULT_PROVIDER_OPTIONS; + const allowed = item.model?.getProviders() || DEFAULT_PROVIDER_OPTIONS; if (allowed.length > 0) { setSelectedProvider(page, modelId, allowed[0]); } @@ -146,11 +143,7 @@ export function ClassyModelSelector({ () => items.map((item) => { const modelId = item.model?.id; - const allowedProviders: Provider[] = modelId - ? (providersByModel?.[modelId] ?? - PROVIDER_LOOKUP_BY_PAGE[page]?.[modelId] ?? - DEFAULT_PROVIDER_OPTIONS) - : DEFAULT_PROVIDER_OPTIONS; + const allowedProviders = item.model?.getProviders() || DEFAULT_PROVIDER_OPTIONS; const hasMultipleProviders = allowedProviders.length >= 2; diff --git a/frontend/libs/components/model-selector/src/lib/provider-icons.tsx b/frontend/libs/components/model-selector/src/lib/provider-icons.tsx index 67b971f9ec..b23df3a060 100644 --- a/frontend/libs/components/model-selector/src/lib/provider-icons.tsx +++ b/frontend/libs/components/model-selector/src/lib/provider-icons.tsx @@ -1,21 +1,21 @@ import { getCreatorIcon, ModelCreator } from "@storyteller/model-list"; import { IsDesktopApp } from "@storyteller/tauri-utils"; -import { Provider } from "@storyteller/tauri-api"; +import { GenerationProvider } from "@storyteller/api-enums"; import { ReactNode } from "react"; -const PROVIDER_TO_CREATOR: Partial> = { - [Provider.ArtCraft]: ModelCreator.ArtCraft, - [Provider.Fal]: ModelCreator.Fal, - [Provider.Sora]: ModelCreator.OpenAi, - [Provider.Grok]: ModelCreator.Grok, - [Provider.Midjourney]: ModelCreator.Midjourney, +const GENERATION_PROVIDER_TO_CREATOR: Partial> = { + [GenerationProvider.Artcraft]: ModelCreator.ArtCraft, + [GenerationProvider.Grok]: ModelCreator.Grok, + [GenerationProvider.Midjourney]: ModelCreator.Midjourney, + [GenerationProvider.Sora]: ModelCreator.OpenAi, + [GenerationProvider.WorldLabs]: ModelCreator.WorldLabs, }; export const getProviderIcon = ( - provider: Provider, + provider: GenerationProvider, className = "h-4 w-4 icon-auto-contrast" ): ReactNode => { - const creator = PROVIDER_TO_CREATOR[provider]; + const creator = GENERATION_PROVIDER_TO_CREATOR[provider]; if (creator) return getCreatorIcon(creator, className); return ( { +export const getProviderDisplayName = (provider: GenerationProvider): string => { switch (provider) { - case Provider.ArtCraft: + case GenerationProvider.Artcraft: return "ArtCraft"; - case Provider.Fal: - return "Fal"; - case Provider.Sora: - return "Sora / ChatGPT"; - case Provider.Grok: + case GenerationProvider.Grok: return "Grok"; - case Provider.Midjourney: + case GenerationProvider.Midjourney: return "Midjourney"; - case "worldlabs": + case GenerationProvider.Sora: + return "Sora / ChatGPT"; + case GenerationProvider.WorldLabs: return "World Labs"; default: - return provider; + return "Unknown Provider"; } }; diff --git a/frontend/libs/components/model-selector/src/lib/provider-lookup.ts b/frontend/libs/components/model-selector/src/lib/provider-lookup.ts deleted file mode 100644 index 844ea5f13d..0000000000 --- a/frontend/libs/components/model-selector/src/lib/provider-lookup.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { ModelPage } from "./model-pages"; -import { Provider } from "@storyteller/tauri-api"; - -export type ProvidersByModel = Partial>; - -// Centralized lookup: page (product/tab) → modelId → allowed providers -export const PROVIDER_LOOKUP_BY_PAGE: Record = { - [ModelPage.TextToImage]: { - gpt_image_1: [Provider.ArtCraft, Provider.Sora], - grok_image: [Provider.Grok], - midjourney: [Provider.Midjourney], - }, - [ModelPage.ImageToVideo]: { - grok_video: [Provider.Grok], - }, - [ModelPage.Canvas2D]: { - gpt_image_1: [Provider.ArtCraft, Provider.Sora], - }, - [ModelPage.Stage3D]: { gpt_image_1: [Provider.ArtCraft, Provider.Sora] }, - [ModelPage.ImageEditor]: { - gpt_image_1: [Provider.ArtCraft, Provider.Sora], - }, -}; diff --git a/frontend/libs/components/model-selector/tsconfig.json b/frontend/libs/components/model-selector/tsconfig.json index ba36f9dc8a..dba6235e59 100644 --- a/frontend/libs/components/model-selector/tsconfig.json +++ b/frontend/libs/components/model-selector/tsconfig.json @@ -11,6 +11,9 @@ { "path": "../../tauri-api" }, + { + "path": "../../api-enums" + }, { "path": "../../model-list" }, diff --git a/frontend/libs/components/model-selector/tsconfig.lib.json b/frontend/libs/components/model-selector/tsconfig.lib.json index 3aa719f743..558e0dc604 100644 --- a/frontend/libs/components/model-selector/tsconfig.lib.json +++ b/frontend/libs/components/model-selector/tsconfig.lib.json @@ -52,6 +52,9 @@ { "path": "../../tauri-api/tsconfig.lib.json" }, + { + "path": "../../api-enums/tsconfig.lib.json" + }, { "path": "../../model-list/tsconfig.lib.json" } diff --git a/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx b/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx index 1f3f9808d4..3091041907 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBox2D.tsx @@ -32,6 +32,7 @@ import { EnqueueEditImage, EnqueueEditImageSize, EnqueueEditImageResolution, + EnqueueEditImageRequest, } from "@storyteller/tauri-api"; import { usePrompt2DStore } from "./promptStore"; import { gtagEvent } from "@storyteller/google-analytics"; @@ -39,6 +40,7 @@ import { getCapabilitiesForModel } from "@storyteller/model-list"; import { ImageModel } from "@storyteller/model-list"; import { ImagePromptRow } from "./ImagePromptRow"; import { twMerge } from "tailwind-merge"; +import { GenerationProvider } from "@storyteller/api-enums"; export type AspectRatio = "wide" | "tall" | "square"; @@ -53,6 +55,7 @@ interface PromptBox2DProps { progressCallback: (newState: UploaderState) => void; }) => Promise; selectedImageModel?: ImageModel; + selectedProvider?: GenerationProvider; getCanvasRenderBitmap: () => MaybeCanvasRenderBitmapType; EncodeImageBitmapToBase64: (imageBitmap: ImageBitmap) => Promise; useJobContext: () => JobContextType; @@ -69,6 +72,7 @@ export const PromptBox2D = ({ onEnqueuePressed, onAspectRatioChange, selectedImageModel, + selectedProvider, onFitPressed, }: PromptBox2DProps) => { useSignals(); @@ -272,7 +276,7 @@ export const PromptBox2D = ({ const aspectRatio = getCurrentAspectRatio(); const resolution = getCurrentResolution(); - const generateResponse = await EnqueueEditImage({ + let request : EnqueueEditImageRequest = { model: selectedImageModel, scene_image_media_token: snapshotMediaToken.data!, image_media_tokens: referenceImages @@ -283,7 +287,13 @@ export const PromptBox2D = ({ image_count: generationCount, aspect_ratio: aspectRatio, image_resolution: resolution, - }); + }; + + if (!!selectedProvider) { + request.provider = selectedProvider; + } + + const generateResponse = await EnqueueEditImage(request); console.log("generateResponse", generateResponse); }; diff --git a/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx b/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx index 8920feb7bf..9257caeafd 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx @@ -42,10 +42,12 @@ import { EnqueueEditImage, EnqueueEditImageSize, EnqueueEditImageResolution, + EnqueueEditImageRequest, } from "@storyteller/tauri-api"; import { usePrompt3DStore } from "./promptStore"; import { gtagEvent } from "@storyteller/google-analytics"; import { ImageModel } from "@storyteller/model-list"; +import { GenerationProvider } from "@storyteller/api-enums"; import { ImagePromptRow } from "./ImagePromptRow"; interface PromptBox3DProps { @@ -67,6 +69,7 @@ interface PromptBox3DProps { onAspectRatioSelect: (newRatio: CameraAspectRatio) => void; setEnginePrompt: (prompt: string) => void; selectedImageModel?: ImageModel; + selectedProvider?: GenerationProvider; snapshotCurrentFrame: | ((shouldDownload?: boolean) => { base64Snapshot: string; @@ -94,6 +97,7 @@ export const PromptBox3D = ({ onAspectRatioSelect, setEnginePrompt, selectedImageModel, + selectedProvider, snapshotCurrentFrame, }: PromptBox3DProps) => { useSignals(); @@ -366,7 +370,7 @@ export const PromptBox3D = ({ const aspectRatio = getCurrentAspectRatio(); const resolution = getCurrentResolution(); - const generateResponse = await EnqueueEditImage({ + let request: EnqueueEditImageRequest = { model: selectedImageModel, scene_image_media_token: snapshotResult.data!, image_media_tokens: referenceImages.map((image) => image.mediaToken), @@ -375,7 +379,13 @@ export const PromptBox3D = ({ image_count: 1, aspect_ratio: aspectRatio, image_resolution: resolution, - }); + }; + + if (!!selectedProvider) { + request.provider = selectedProvider; + } + + const generateResponse = await EnqueueEditImage(request); console.log("generateResponse", generateResponse); diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxEdit.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxEdit.tsx index 63931acad7..bddd273154 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBoxEdit.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxEdit.tsx @@ -29,6 +29,7 @@ import { ImageModel, getCapabilitiesForModel } from "@storyteller/model-list"; import { twMerge } from "tailwind-merge"; import { ImagePromptRow, type UploadImageFn } from "./ImagePromptRow"; import { RefImage, usePromptEditStore } from "./promptStore"; +import { GenerationProvider } from "@storyteller/api-enums"; export interface PromptBoxEditProps { onModeChange?: (mode: string) => void; @@ -39,11 +40,13 @@ export interface PromptBoxEditProps { aspectRatio?: string; resolution?: string; images?: RefImage[]; + selectedProvider?: GenerationProvider; }, ) => void | Promise; isDisabled?: boolean; onFitPressed?: () => void | Promise; selectedImageModel?: ImageModel; + selectedProvider?: GenerationProvider; generationCount?: number; onGenerationCountChange?: (count: number) => void; supportsMaskedInpainting?: boolean; @@ -62,6 +65,7 @@ export const PromptBoxEdit = ({ isDisabled, onFitPressed, selectedImageModel, + selectedProvider, generationCount: generationCountProp, onGenerationCountChange, supportsMaskedInpainting = false, @@ -277,6 +281,7 @@ export const PromptBoxEdit = ({ aspectRatio, resolution, images: referenceImages, + selectedProvider: selectedProvider, }), ); } finally { diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx index c8f0a3ee93..d2780b2af4 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx @@ -32,6 +32,7 @@ import { usePromptImageStore, RefImage } from "./promptStore"; import { gtagEvent } from "@storyteller/google-analytics"; import { twMerge } from "tailwind-merge"; import { ImagePromptRow } from "./ImagePromptRow"; +import { GenerationProvider } from "@storyteller/api-enums"; interface PromptBoxImageProps { useJobContext: () => JobContextType; @@ -50,6 +51,7 @@ interface PromptBoxImageProps { subscriberId: string, ) => void | Promise; selectedModel?: ImageModel; + selectedProvider?: GenerationProvider; imageMediaId?: string; url?: string; onImageRowVisibilityChange?: (visible: boolean) => void; @@ -60,6 +62,7 @@ export const PromptBoxImage = ({ uploadImage, onEnqueuePressed, selectedModel, + selectedProvider, imageMediaId, url, onImageRowVisibilityChange, @@ -299,6 +302,10 @@ export const PromptBoxImage = ({ frontend_subscriber_id: subscriberId, }; + if (!!selectedProvider) { + request.provider = selectedProvider; + } + if ( selectedModel?.canUseImagePrompt && !!referenceImages && diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx index 58f3ff9a29..7e57a2d0e9 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx @@ -36,6 +36,7 @@ import { ImagePromptRow } from "./ImagePromptRow"; import type { UploadImageFn } from "./ImagePromptRow"; import { twMerge } from "tailwind-merge"; import { toast } from "@storyteller/ui-toaster"; +import { GenerationProvider } from "@storyteller/api-enums"; type GROK_ASPECT_RATIO = "landscape" | "portrait" | "square"; @@ -59,6 +60,7 @@ interface PromptBoxVideoProps { subscriberId: string, ) => void | Promise; selectedModel?: VideoModel; + selectedProvider?: GenerationProvider; imageMediaId?: string; url?: string; onImageRowVisibilityChange?: (visible: boolean) => void; @@ -69,6 +71,7 @@ export const PromptBoxVideo = ({ useJobContext, onEnqueuePressed, selectedModel, + selectedProvider, imageMediaId, url, onImageRowVisibilityChange, @@ -285,6 +288,10 @@ export const PromptBoxVideo = ({ frontend_subscriber_id: subscriberId, }; + if (!!selectedProvider) { + request.provider = selectedProvider; + } + if (selectedModel.generateWithSound) { request.generate_audio = !!generateWithSound; } diff --git a/frontend/libs/model-list/src/lib/classes/ImageModel.ts b/frontend/libs/model-list/src/lib/classes/ImageModel.ts index 8270c298eb..a21f0880c7 100644 --- a/frontend/libs/model-list/src/lib/classes/ImageModel.ts +++ b/frontend/libs/model-list/src/lib/classes/ImageModel.ts @@ -2,6 +2,7 @@ import { ModelCreator } from "src/index.js"; import { Model, ModelKind } from "./Model.js"; import { ModelCategory } from "../legacy/ModelConfig.js"; import { ModelTag } from "./metadata/ModelTag.js"; +import { GenerationProvider } from "@storyteller/api-enums"; export class ImageModel extends Model { // Typescript type discriminator property @@ -63,6 +64,7 @@ export class ImageModel extends Model { canChangeAspectRatio?: boolean; tags?: ModelTag[]; progressBarTime?: number; + providers?: GenerationProvider[]; }) { if (args.maxGenerationCount < 1) { throw new Error("maxGenerationCount must be at least 1"); diff --git a/frontend/libs/model-list/src/lib/classes/Model.ts b/frontend/libs/model-list/src/lib/classes/Model.ts index 8ef8190ecd..fc6d378d82 100644 --- a/frontend/libs/model-list/src/lib/classes/Model.ts +++ b/frontend/libs/model-list/src/lib/classes/Model.ts @@ -1,6 +1,7 @@ import { ModelCreator } from "./metadata/ModelCreator.js"; import { ModelCategory, ModelConfig } from "../legacy/ModelConfig.js"; import { ModelTag } from "./metadata/ModelTag.js"; +import { GenerationProvider } from "@storyteller/api-enums"; export type ModelKind = "model" | "image_model" | "video_model" | "gaussian_model"; @@ -40,6 +41,9 @@ export class Model { // A list of filterable "capabilities" that can be used to filter models. readonly tags: ModelTag[]; + // Use `getProviders()` to read the list. + protected readonly providers?: GenerationProvider[]; + // Optional provider preferences; two-to-many via page and providers. readonly preferredProvidersByPage?: Partial>; @@ -56,6 +60,7 @@ export class Model { selectorDescription: string; selectorBadges: string[]; tags?: ModelTag[]; + providers?: GenerationProvider[]; preferredProvidersByPage?: Partial>; progressBarTime?: number; }) { @@ -68,10 +73,15 @@ export class Model { this.selectorDescription = args.selectorDescription; this.selectorBadges = args.selectorBadges; this.tags = args.tags ?? []; + this.providers = args.providers; this.preferredProvidersByPage = args.preferredProvidersByPage; this.progressBarTime = args.progressBarTime ?? 20000; } + getProviders(): GenerationProvider[] { + return this.providers ?? [GenerationProvider.Artcraft]; + } + toLegacyBadges(): { label: string }[] { return this.selectorBadges.map((b) => ({ label: b })); } diff --git a/frontend/libs/model-list/src/lib/classes/VideoModel.ts b/frontend/libs/model-list/src/lib/classes/VideoModel.ts index 771e35f375..0b506370ad 100644 --- a/frontend/libs/model-list/src/lib/classes/VideoModel.ts +++ b/frontend/libs/model-list/src/lib/classes/VideoModel.ts @@ -3,6 +3,7 @@ import { Model, ModelKind } from "./Model.js"; import { ModelCategory } from "../legacy/ModelConfig.js"; import { ModelTag } from "./metadata/ModelTag.js"; import { SizeOption } from "./metadata/SizeOption.js"; +import { GenerationProvider } from "@storyteller/api-enums"; export class VideoModel extends Model { // Typescript type discriminator property @@ -41,6 +42,7 @@ export class VideoModel extends Model { sizeOptions?: SizeOption[]; progressBarTime?: number; generateWithSound?: boolean; + providers?: GenerationProvider[]; }) { super(args); this.startFrame = args.startFrame; diff --git a/frontend/libs/model-list/src/lib/lists/ImageModels.ts b/frontend/libs/model-list/src/lib/lists/ImageModels.ts index d0f6ff088e..c09831605c 100644 --- a/frontend/libs/model-list/src/lib/lists/ImageModels.ts +++ b/frontend/libs/model-list/src/lib/lists/ImageModels.ts @@ -1,3 +1,4 @@ +import { GenerationProvider } from "@storyteller/api-enums"; import { ImageModel } from "../classes/ImageModel.js"; import { ModelCreator } from "../classes/metadata/ModelCreator.js"; import { ModelTag } from "../classes/metadata/ModelTag.js"; @@ -11,6 +12,9 @@ export const IMAGE_MODELS: ImageModel[] = [ fullName: "Grok", category: "image", creator: ModelCreator.Grok, + providers: [ + GenerationProvider.Grok, + ], selectorName: "Grok", selectorDescription: "Fast af", selectorBadges: ["10 sec."], @@ -27,6 +31,9 @@ export const IMAGE_MODELS: ImageModel[] = [ fullName: "Midjourney", category: "image", creator: ModelCreator.Midjourney, + providers: [ + GenerationProvider.Midjourney + ], selectorName: "Midjourney", selectorDescription: "Stunning style and quality", selectorBadges: ["45 sec."], @@ -147,6 +154,10 @@ export const IMAGE_MODELS: ImageModel[] = [ fullName: "GPT Image 1 (GPT-4o)", category: "image", creator: ModelCreator.OpenAi, + providers: [ + GenerationProvider.Artcraft, + GenerationProvider.Sora + ], selectorName: "GPT Image 1 (GPT-4o)", selectorDescription: "Slow, but super smart", selectorBadges: ["60 sec."], diff --git a/frontend/libs/model-list/src/lib/lists/VideoModels.ts b/frontend/libs/model-list/src/lib/lists/VideoModels.ts index 47e315c916..5c56b23baa 100644 --- a/frontend/libs/model-list/src/lib/lists/VideoModels.ts +++ b/frontend/libs/model-list/src/lib/lists/VideoModels.ts @@ -1,3 +1,4 @@ +import { GenerationProvider } from "@storyteller/api-enums"; import { VideoModel } from "../classes/VideoModel.js"; import { ModelCreator } from "../classes/metadata/ModelCreator.js"; import { SizeIconOption } from "../classes/metadata/SizeOption.js"; @@ -9,6 +10,9 @@ export const VIDEO_MODELS: VideoModel[] = [ fullName: "Grok Video", category: "video", creator: ModelCreator.Grok, + providers: [ + GenerationProvider.Grok + ], selectorName: "Grok Video", selectorDescription: "Fastest video model", selectorBadges: ["20 sec."], @@ -129,6 +133,10 @@ export const VIDEO_MODELS: VideoModel[] = [ fullName: "Sora 2", category: "video", creator: ModelCreator.OpenAi, + providers: [ + GenerationProvider.Artcraft, + GenerationProvider.Sora, + ], selectorName: "Sora 2", selectorDescription: "Smart video model", selectorBadges: ["2 min."], diff --git a/frontend/libs/model-list/tsconfig.json b/frontend/libs/model-list/tsconfig.json index 85fee3a884..974f602415 100644 --- a/frontend/libs/model-list/tsconfig.json +++ b/frontend/libs/model-list/tsconfig.json @@ -6,6 +6,9 @@ { "path": "../tauri-utils" }, + { + "path": "../api-enums" + }, { "path": "./tsconfig.lib.json" }, diff --git a/frontend/libs/model-list/tsconfig.lib.json b/frontend/libs/model-list/tsconfig.lib.json index 7abf32a011..806620a109 100644 --- a/frontend/libs/model-list/tsconfig.lib.json +++ b/frontend/libs/model-list/tsconfig.lib.json @@ -20,6 +20,9 @@ "references": [ { "path": "../tauri-utils/tsconfig.lib.json" + }, + { + "path": "../api-enums/tsconfig.lib.json" } ], "exclude": [ diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts index f870fee7e6..536c5a1c71 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts @@ -1,12 +1,15 @@ import { invoke } from "@tauri-apps/api/core"; import { CommandResult } from "../common/CommandStatus"; import { ImageModel } from "@storyteller/model-list"; +import { GenerationProvider } from "@storyteller/api-enums"; export interface EnqueueEditImageRequest { - /// The model to use. model?: ImageModel | EnqueueEditImageModel; + // The provider to use. + provider?: GenerationProvider; + /// If set, this is the first image in the contextual image set. /// This gets submitted along with `image_media_tokens` and will // be prompt engineered by Tauri. @@ -45,6 +48,7 @@ export interface EnqueueEditImageRequest { // (We do some transformations from the public-facing request object.) interface RawEnqueueEditImageRequest { model?: EnqueueEditImageModel | string; + provider?: GenerationProvider; scene_image_media_token?: string; image_media_tokens?: string[]; prompt?: string; @@ -126,6 +130,10 @@ export const EnqueueEditImage = async (request: EnqueueEditImageRequest) : Promi model: modelName, prompt: request.prompt, }; + + if (!!request.provider) { + mutableRequest.provider = request.provider; + } if (!!request.scene_image_media_token) { mutableRequest.scene_image_media_token = request.scene_image_media_token; diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts index 91c3c2c241..22b0eca42f 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts @@ -1,11 +1,15 @@ import { invoke } from "@tauri-apps/api/core"; import { CommandResult } from "../common/CommandStatus"; import { ImageModel, ModelInfo } from "@storyteller/model-list"; +import { GenerationProvider } from "@storyteller/api-enums"; export interface EnqueueImageInpaintRequest { // The model to use. model?: ImageModel | ModelInfo | EnqueueImageInpaintModel; + // The provider to use. + provider?: GenerationProvider; + // The image we're editing. image_media_token?: string; @@ -30,6 +34,7 @@ export interface EnqueueImageInpaintRequest { // (We do some transformations from the public-facing request object.) interface RawEnqueueImageInpaintRequest { model?: EnqueueImageInpaintModel | string; + provider?: GenerationProvider; image_media_token?: string; mask_image_raw_bytes?: Uint8Array; prompt?: string; @@ -89,6 +94,10 @@ export const EnqueueImageInpaint = async (request: EnqueueImageInpaintRequest) : model: modelName, prompt: request.prompt, }; + + if (!!request.provider) { + mutableRequest.provider = request.provider; + } if (!!request.image_media_token) { mutableRequest.image_media_token = request.image_media_token; diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToVideo.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToVideo.ts index c1777bc6db..e461ef8f6b 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToVideo.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageToVideo.ts @@ -1,6 +1,7 @@ import { invoke } from "@tauri-apps/api/core"; import { CommandResult } from "../common/CommandStatus"; import { Model } from "@storyteller/model-list"; +import { GenerationProvider } from "@storyteller/api-enums"; export enum EnqueueImageToVideoErrorType { /// Caller didn't specify a model @@ -17,6 +18,9 @@ export interface EnqueueImageToVideoRequest { // Required. The model to use. model?: Model; + // The provider to use. + provider?: GenerationProvider; + // Required. Starting frame. image_media_token?: string; @@ -44,6 +48,7 @@ export interface EnqueueImageToVideoRequest { interface RawEnqueueImageToVideoRequest { model?: string; + provider?: GenerationProvider; image_media_token?: string; end_frame_image_media_token?: string; prompt?: string; @@ -77,6 +82,10 @@ export const EnqueueImageToVideo = async ( image_media_token: request.image_media_token, }; + if (request.provider) { + mutableRequest.provider = request.provider; + } + if (request.prompt) { mutableRequest.prompt = request.prompt; } diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts index 75aad0dbd3..99a13a1077 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts @@ -1,6 +1,7 @@ import { invoke } from "@tauri-apps/api/core"; import { CommandResult } from "../common/CommandStatus"; import { Model } from "@storyteller/model-list"; +import { GenerationProvider } from "@storyteller/api-enums"; export enum EnqueueTextToImageErrorType { /// Caller didn't specify a model @@ -17,6 +18,9 @@ export interface EnqueueTextToImageRequest { // The model to use. model?: Model | EnqueueTextToImageModel; + // The provider to use. + provider?: GenerationProvider; + // The text prompt. prompt?: string; @@ -46,8 +50,9 @@ export interface EnqueueTextToImageRequest { // Shape of request sent to Tauri // (We do some transformations from the public-facing request object.) interface EnqueueTextToImageRawRequest { - prompt?: string; model?: EnqueueTextToImageModel | string; // TODO: Shouldn't allow string + provider?: GenerationProvider; + prompt?: string; aspect_ratio?: EnqueueTextToImageSize; image_resolution?: EnqueueTextToImageResolution; number_images?: number; @@ -113,6 +118,10 @@ export const EnqueueTextToImage = async (request: EnqueueTextToImageRequest) : P prompt: request.prompt, }; + if (!!request.provider) { + mutableRequest.provider = request.provider; + } + if (!!request.aspect_ratio) { mutableRequest.aspect_ratio = request.aspect_ratio; } diff --git a/frontend/libs/tauri-api/src/lib/providers/Provider.ts b/frontend/libs/tauri-api/src/lib/providers/Provider.ts index ec3587a8c5..1d6f451fad 100644 --- a/frontend/libs/tauri-api/src/lib/providers/Provider.ts +++ b/frontend/libs/tauri-api/src/lib/providers/Provider.ts @@ -1,9 +1,10 @@ +// TODO: REMOVE THIS. +// Deprecated in favor of @storyteller/api-enums - GenerationProvider export enum Provider { ArtCraft = "artcraft", Fal = "fal", Sora = "sora", Grok = "grok", Midjourney = "midjourney", - }