adjust frontend for omni models

This commit is contained in:
Brandon Thomas committed 2026-07-01 16:32:16 -04:00
1 parent a61fcd50ed
commit 5cd71a0db6
16 files changed
+701 -158

No files matched your search

+8
View File
@@ -14,6 +14,7 @@ import { pageHeight, pageWidth, persistLogin } from "~/signals";
import { SyncStorytellerApiConfig } from "./api/SyncStorytellerApiConfig";
import { posthog } from "posthog-js";
import { SoundManager } from "@storyteller/soundboard";
import { useModelsStore } from "@storyteller/tauri-api";
config.autoAddCss = false; /* eslint-disable import/first */
@@ -89,6 +90,13 @@ const GlobalSettingsManager = ({ env }: { env: Record<string, string> }) => {
SoundManager.install();
}, []);
// Reconcile the model dropdowns against the backend omni listing once on boot.
// The store is already seeded with the static overlay, so a failure here is a
// no-op (the UI keeps the overlay models).
useEffect(() => {
void useModelsStore.getState().loadModelsFromBackend();
}, []);
return null;
};
@@ -37,7 +37,7 @@ import { SliderV2 } from "@storyteller/ui-sliderv2";
import { GenerateImage } from "@storyteller/tauri-api";
import {
ClassyModelSelector,
ANGLES_PAGE_MODEL_LIST,
useAnglesPageModelList,
ModelPage,
useSelectedModel,
useSelectedProviderForModel,
@@ -58,6 +58,7 @@ const ANGLES_MODEL_PAGE = ModelPage.Angles;
// ─── Main Angles Component ─────────────────────────────────────────────────────
export const Angles = () => {
const anglesModelList = useAnglesPageModelList();
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
[],
@@ -733,7 +734,7 @@ export const Angles = () => {
<ClassyModelSelector
variant="embedded"
items={ANGLES_PAGE_MODEL_LIST}
items={anglesModelList}
page={ANGLES_MODEL_PAGE}
/>
<GenerateButton
@@ -4,7 +4,7 @@ import { PromptBoxImage } from "@storyteller/ui-promptbox";
import { UploadImageMedia, FilterMediaClasses } from "@storyteller/api";
import BackgroundGallery from "./BackgroundGallery";
import {
TEXT_TO_IMAGE_PAGE_MODEL_LIST,
useTextToImagePageModelList,
ModelPage,
ClassyModelSelector,
useSelectedImageModel,
@@ -44,6 +44,7 @@ interface TextToImageProps {
}
const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
const textToImageModelList = useTextToImagePageModelList();
const startBatch = useTextToImageStore((s) => s.startBatch);
const promptContentRef = useRef<HTMLDivElement>(null);
const [promptHeight, setPromptHeight] = useState<number>(138);
@@ -187,7 +188,7 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
modelSelector={
<ClassyModelSelector
variant="embedded"
items={TEXT_TO_IMAGE_PAGE_MODEL_LIST}
items={textToImageModelList}
page={PAGE_ID}
/>
}
@@ -13,7 +13,7 @@ import {
import BackgroundGallery from "./BackgroundGallery";
import {
ClassyModelSelector,
IMAGE_TO_VIDEO_PAGE_MODEL_LIST,
useImageToVideoPageModelList,
ModelPage,
useSelectedVideoModel,
useSelectedProviderForModel,
@@ -56,6 +56,7 @@ interface ImageToVideoProps {
}
const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => {
const imageToVideoModelList = useImageToVideoPageModelList();
const startBatch = useImageToVideoStore((s) => s.startBatch);
const completeBatch = useImageToVideoStore((s) => s.completeBatch);
const promptContentRef = useRef<HTMLDivElement>(null);
@@ -215,7 +216,7 @@ const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => {
modelSelector={
<ClassyModelSelector
variant="embedded"
items={IMAGE_TO_VIDEO_PAGE_MODEL_LIST}
items={imageToVideoModelList}
page={PAGE_ID}
/>
}
@@ -0,0 +1 @@
{"fileNames":["../../node_modules/typescript/lib/lib.es5.d.ts","../../node_modules/typescript/lib/lib.es2015.d.ts","../../node_modules/typescript/lib/lib.es2016.d.ts","../../node_modules/typescript/lib/lib.es2017.d.ts","../../node_modules/typescript/lib/lib.es2018.d.ts","../../node_modules/typescript/lib/lib.es2019.d.ts","../../node_modules/typescript/lib/lib.es2020.d.ts","../../node_modules/typescript/lib/lib.es2021.d.ts","../../node_modules/typescript/lib/lib.es2022.d.ts","../../node_modules/typescript/lib/lib.es2023.d.ts","../../node_modules/typescript/lib/lib.es2024.d.ts","../../node_modules/typescript/lib/lib.esnext.d.ts","../../node_modules/typescript/lib/lib.dom.d.ts","../../node_modules/typescript/lib/lib.dom.iterable.d.ts","../../node_modules/typescript/lib/lib.es2015.core.d.ts","../../node_modules/typescript/lib/lib.es2015.collection.d.ts","../../node_modules/typescript/lib/lib.es2015.generator.d.ts","../../node_modules/typescript/lib/lib.es2015.iterable.d.ts","../../node_modules/typescript/lib/lib.es2015.promise.d.ts","../../node_modules/typescript/lib/lib.es2015.proxy.d.ts","../../node_modules/typescript/lib/lib.es2015.reflect.d.ts","../../node_modules/typescript/lib/lib.es2015.symbol.d.ts","../../node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts","../../node_modules/typescript/lib/lib.es2016.array.include.d.ts","../../node_modules/typescript/lib/lib.es2016.intl.d.ts","../../node_modules/typescript/lib/lib.es2017.arraybuffer.d.ts","../../node_modules/typescript/lib/lib.es2017.date.d.ts","../../node_modules/typescript/lib/lib.es2017.object.d.ts","../../node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts","../../node_modules/typescript/lib/lib.es2017.string.d.ts","../../node_modules/typescript/lib/lib.es2017.intl.d.ts","../../node_modules/typescript/lib/lib.es2017.typedarrays.d.ts","../../node_modules/typescript/lib/lib.es2018.asyncgenerator.d.ts","../../node_modules/typescript/lib/lib.es2018.asynciterable.d.ts","../../node_modules/typescript/lib/lib.es2018.intl.d.ts","../../node_modules/typescript/lib/lib.es2018.promise.d.ts","../../node_modules/typescript/lib/lib.es2018.regexp.d.ts","../../node_modules/typescript/lib/lib.es2019.array.d.ts","../../node_modules/typescript/lib/lib.es2019.object.d.ts","../../node_modules/typescript/lib/lib.es2019.string.d.ts","../../node_modules/typescript/lib/lib.es2019.symbol.d.ts","../../node_modules/typescript/lib/lib.es2019.intl.d.ts","../../node_modules/typescript/lib/lib.es2020.bigint.d.ts","../../node_modules/typescript/lib/lib.es2020.date.d.ts","../../node_modules/typescript/lib/lib.es2020.promise.d.ts","../../node_modules/typescript/lib/lib.es2020.sharedmemory.d.ts","../../node_modules/typescript/lib/lib.es2020.string.d.ts","../../node_modules/typescript/lib/lib.es2020.symbol.wellknown.d.ts","../../node_modules/typescript/lib/lib.es2020.intl.d.ts","../../node_modules/typescript/lib/lib.es2020.number.d.ts","../../node_modules/typescript/lib/lib.es2021.promise.d.ts","../../node_modules/typescript/lib/lib.es2021.string.d.ts","../../node_modules/typescript/lib/lib.es2021.weakref.d.ts","../../node_modules/typescript/lib/lib.es2021.intl.d.ts","../../node_modules/typescript/lib/lib.es2022.array.d.ts","../../node_modules/typescript/lib/lib.es2022.error.d.ts","../../node_modules/typescript/lib/lib.es2022.intl.d.ts","../../node_modules/typescript/lib/lib.es2022.object.d.ts","../../node_modules/typescript/lib/lib.es2022.string.d.ts","../../node_modules/typescript/lib/lib.es2022.regexp.d.ts","../../node_modules/typescript/lib/lib.es2023.array.d.ts","../../node_modules/typescript/lib/lib.es2023.collection.d.ts","../../node_modules/typescript/lib/lib.es2023.intl.d.ts","../../node_modules/typescript/lib/lib.es2024.arraybuffer.d.ts","../../node_modules/typescript/lib/lib.es2024.collection.d.ts","../../node_modules/typescript/lib/lib.es2024.object.d.ts","../../node_modules/typescript/lib/lib.es2024.promise.d.ts","../../node_modules/typescript/lib/lib.es2024.regexp.d.ts","../../node_modules/typescript/lib/lib.es2024.sharedmemory.d.ts","../../node_modules/typescript/lib/lib.es2024.string.d.ts","../../node_modules/typescript/lib/lib.esnext.array.d.ts","../../node_modules/typescript/lib/lib.esnext.collection.d.ts","../../node_modules/typescript/lib/lib.esnext.intl.d.ts","../../node_modules/typescript/lib/lib.esnext.disposable.d.ts","../../node_modules/typescript/lib/lib.esnext.promise.d.ts","../../node_modules/typescript/lib/lib.esnext.decorators.d.ts","../../node_modules/typescript/lib/lib.esnext.iterator.d.ts","../../node_modules/typescript/lib/lib.esnext.float16.d.ts","../../node_modules/typescript/lib/lib.decorators.d.ts","../../node_modules/typescript/lib/lib.decorators.legacy.d.ts","../../node_modules/@remix-run/node/dist/globals.d.ts","../../node_modules/@types/cookie/index.d.ts","../../node_modules/@remix-run/server-runtime/dist/crypto.d.ts","../../node_modules/@remix-run/server-runtime/dist/cookies.d.ts","../../node_modules/@remix-run/server-runtime/dLine truncated
@@ -1,24 +1,25 @@
import { useMemo } from "react";
import type { PopoverItem } from "@storyteller/ui-popover";
import { faCube, faFilm, faImage } from "@fortawesome/pro-solid-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
getCreatorIcon,
Model,
ImageModel,
VideoModel,
VIDEO_MODELS,
IMAGE_MODELS,
SPLAT_MODELS,
OBJECT_3D_MODELS,
} from "@storyteller/model-list";
import { ModelTag } from "@storyteller/model-list";
import { useImageModels, useVideoModels } from "@storyteller/tauri-api";
export type ModelList = Omit<PopoverItem, "selected">[];
const withIcon = (creatorIcon: any, fallback: any) => creatorIcon || fallback;
const buildItems = (
models: Model[],
fallbackIcon: any
) =>
const buildItems = (models: Model[], fallbackIcon: any) =>
models.map((model: Model) => ({
label: model.selectorName,
icon: withIcon(getCreatorIcon(model.creator), fallbackIcon),
@@ -27,113 +28,137 @@ const buildItems = (
model: model,
}));
const sortedBySelectorName = <T extends Model>(models: T[]): T[] => {
// De-dupe while preserving instances, then sort by selector name.
const list = Array.from(new Set(models));
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
};
const imageIcon = <FontAwesomeIcon icon={faImage} className="h-4 w-4" />;
const filmIcon = <FontAwesomeIcon icon={faFilm} className="h-4 w-4" />;
const cubeIcon = <FontAwesomeIcon icon={faCube} className="h-4 w-4" />;
/**
* IN-PROGRESS MIGRATION (messy for now)
* We're gradually going to phase out ModelList, ModelConfig, etc.
* We won't index by name, but rather id, or simply will always have full
* access to the object directly.
* Per-page model subsetting. These pages show a slice of the full model set
* based on model CAPABILITY FLAGS (canTextToImage, canEditImages, canEditAngles,
* the InstructiveEdit tag, etc.). Those flags do not exist in the storyteller-web
* omni backend, so this subsetting is maintained here in the frontend overlay.
*
* Each builder is a pure function over a model list, so it can run against either
* the static overlay (the `*_PAGE_MODEL_LIST` constants below) or the live,
* backend-reconciled list from `useModelsStore` (the `use*PageModelList` hooks).
*/
export const TEXT_TO_IMAGE_PAGE_MODEL_LIST: ModelList =
export const buildTextToImagePageList = (imageModels: ImageModel[]): ModelList =>
buildItems(
(function (): Model[] {
const set: Set<Model> = new Set();
IMAGE_MODELS
.filter((model) => model.canTextToImage)
.forEach((m) => set.add(m));
const list = Array.from(set);
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
})(),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
sortedBySelectorName(imageModels.filter((m) => m.canTextToImage)),
imageIcon,
);
export const buildCanvas2dPageList = (imageModels: ImageModel[]): ModelList =>
buildItems(
sortedBySelectorName(
imageModels.filter(
(m) => m.canEditImages || m.tags?.includes(ModelTag.InstructiveEdit),
),
),
imageIcon,
);
export const buildStage3dPageList = (imageModels: ImageModel[]): ModelList =>
buildItems(
sortedBySelectorName(
imageModels.filter((m) => m.tags?.includes(ModelTag.InstructiveEdit)),
),
imageIcon,
);
export const buildImageEditorPageList = (imageModels: ImageModel[]): ModelList =>
buildItems(
sortedBySelectorName(imageModels.filter((m) => m.canEditImages)),
imageIcon,
);
export const buildAnglesPageList = (imageModels: ImageModel[]): ModelList =>
buildItems(
sortedBySelectorName(imageModels.filter((m) => m.canEditAngles)),
imageIcon,
);
export const buildImageToVideoPageList = (videoModels: VideoModel[]): ModelList =>
buildItems(
// SwitchX is a VFX/background-change model with its own page; keep it
// out of the general video selector.
sortedBySelectorName(videoModels.filter((m) => m.id !== "switch_x")),
filmIcon,
);
// ---------------------------------------------------------------------------
// Backward-compatible static constants (computed from the frontend overlay).
// Prefer the `use*PageModelList` hooks in React components so the dropdowns
// reflect the backend-reconciled model set.
// ---------------------------------------------------------------------------
export const TEXT_TO_IMAGE_PAGE_MODEL_LIST: ModelList =
buildTextToImagePageList(IMAGE_MODELS);
export const CANVAS_2D_PAGE_MODEL_LIST: ModelList =
buildItems(
(function (): Model[] {
const set: Set<Model> = new Set();
IMAGE_MODELS
.filter((m) => m.canEditImages || m.tags?.includes(ModelTag.InstructiveEdit))
.forEach((m) => set.add(m));
const list = Array.from(set);
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
})(),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
buildCanvas2dPageList(IMAGE_MODELS);
export const STAGE_3D_PAGE_MODEL_LIST: ModelList =
buildItems(
(function (): Model[] {
const set: Set<Model> = new Set();
IMAGE_MODELS
.filter((m) => m.tags?.includes(ModelTag.InstructiveEdit))
.forEach((m) => set.add(m));
const list = Array.from(set);
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
})(),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
buildStage3dPageList(IMAGE_MODELS);
export const IMAGE_EDITOR_PAGE_MODEL_LIST: ModelList =
buildItems(
//[
// ALL_MODELS_BY_ID.get("flux_pro_inpaint")!,
// ALL_MODELS_BY_ID.get("flux_dev_juggernaut_inpaint")!,
// ALL_MODELS_BY_ID.get("flux_pro_kontext_max")!,
//],
(function (): Model[] {
const set: Set<Model> = new Set();
//set.add(IMAGE_MODELS_BY_ID.get("gpt_image_1")!); // Place gpt_image_1 first.
IMAGE_MODELS
.filter((m) => m.canEditImages)
.forEach((m) => set.add(m));
const list = Array.from(set);
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
})(),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
buildImageEditorPageList(IMAGE_MODELS);
export const IMAGE_TO_VIDEO_PAGE_MODEL_LIST: ModelList =
buildItems(
(function (): Model[] {
const set: Set<Model> = new Set();
// SwitchX is a VFX/background-change model with its own page; keep it
// out of the general video selector.
VIDEO_MODELS.filter((m) => m.id !== "switch_x").forEach((m) =>
set.add(m),
);
const list = Array.from(set);
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
})(),
<FontAwesomeIcon icon={faFilm} className="h-4 w-4" />
);
buildImageToVideoPageList(VIDEO_MODELS);
export const ANGLES_PAGE_MODEL_LIST: ModelList =
buildItems(
(function (): Model[] {
const set: Set<Model> = new Set();
IMAGE_MODELS
.filter((m) => m.canEditAngles)
.forEach((m) => set.add(m));
const list = Array.from(set);
list.sort((a, b) => a.selectorName?.localeCompare(b.selectorName));
return list;
})(),
<FontAwesomeIcon icon={faImage} className="h-4 w-4" />
);
buildAnglesPageList(IMAGE_MODELS);
export const IMAGE_TO_3D_WORLD_PAGE_MODEL_LIST: ModelList =
buildItems(
SPLAT_MODELS as Model[],
<FontAwesomeIcon icon={faCube} className="h-4 w-4" />
);
export const IMAGE_TO_3D_WORLD_PAGE_MODEL_LIST: ModelList = buildItems(
SPLAT_MODELS as Model[],
cubeIcon,
);
export const IMAGE_TO_3D_OBJECT_PAGE_MODEL_LIST: ModelList =
buildItems(
OBJECT_3D_MODELS as Model[],
<FontAwesomeIcon icon={faCube} className="h-4 w-4" />
);
export const IMAGE_TO_3D_OBJECT_PAGE_MODEL_LIST: ModelList = buildItems(
OBJECT_3D_MODELS as Model[],
cubeIcon,
);
// ---------------------------------------------------------------------------
// Live, backend-reconciled hooks. Use these in React components; they fall back
// to the overlay until `loadModelsFromBackend()` completes at app startup.
// ---------------------------------------------------------------------------
export const useTextToImagePageModelList = (): ModelList => {
const imageModels = useImageModels();
return useMemo(() => buildTextToImagePageList(imageModels), [imageModels]);
};
export const useCanvas2dPageModelList = (): ModelList => {
const imageModels = useImageModels();
return useMemo(() => buildCanvas2dPageList(imageModels), [imageModels]);
};
export const useStage3dPageModelList = (): ModelList => {
const imageModels = useImageModels();
return useMemo(() => buildStage3dPageList(imageModels), [imageModels]);
};
export const useImageEditorPageModelList = (): ModelList => {
const imageModels = useImageModels();
return useMemo(() => buildImageEditorPageList(imageModels), [imageModels]);
};
export const useAnglesPageModelList = (): ModelList => {
const imageModels = useImageModels();
return useMemo(() => buildAnglesPageList(imageModels), [imageModels]);
};
export const useImageToVideoPageModelList = (): ModelList => {
const videoModels = useVideoModels();
return useMemo(() => buildImageToVideoPageList(videoModels), [videoModels]);
};
@@ -24,7 +24,7 @@ import { ContextMenuContainer } from "./components/ui/ContextMenu";
import InpaintToolBar from "./components/ui/InpaintToolBar";
import { ImageModel } from "@storyteller/model-list";
import {
CANVAS_2D_PAGE_MODEL_LIST,
useCanvas2dPageModelList,
ClassyModelSelector,
ModelPage,
useSelectedImageModel,
@@ -293,6 +293,7 @@ const PageDraw = ({
fillParentHeight = false,
showBottomRightControls = true,
}: PageDrawProps) => {
const canvas2dModelList = useCanvas2dPageModelList();
const canvasWidth = useRef<number>(1024);
const canvasHeight = useRef<number>(1024);
const [isSelecting, setIsSelecting] = useState<boolean>(false);
@@ -1215,7 +1216,7 @@ const PageDraw = ({
modelSelector={
<ClassyModelSelector
variant="embedded"
items={CANVAS_2D_PAGE_MODEL_LIST}
items={canvas2dModelList}
page={PAGE_ID}
showProviderSelection={false}
/>
@@ -20,7 +20,7 @@ import {
removeImageDropListener,
} from "@storyteller/ui-gallery-modal";
import {
STAGE_3D_PAGE_MODEL_LIST,
useStage3dPageModelList,
ModelPage,
defaultModelForPage,
useClassyModelSelectorStore,
@@ -109,6 +109,7 @@ export const Stage3DBody = ({
topBarStartSlot,
topBarEndSlot,
}: Stage3DBodyProps = {}) => {
const stage3dModelList = useStage3dPageModelList();
const camAspect = usePageSceneStore((s) => s.cameraAspectRatio);
const outlinerShowing = usePageSceneStore((s) => s.outlinerShowing);
const editorLoader = usePageSceneStore((s) => s.editorLoader);
@@ -165,20 +166,25 @@ export const Stage3DBody = ({
useEffect(() => {
if (modelSelectorPlacement !== "prompt-box") return;
if (selectedImageModel) return;
const models = STAGE_3D_PAGE_MODEL_LIST.map((i) => i.model).filter(
const models = stage3dModelList.map((i) => i.model).filter(
(m): m is NonNullable<typeof m> => m !== undefined,
);
const def = defaultModelForPage(models, PAGE_ID);
if (def) setSelectedModel(PAGE_ID, def);
}, [modelSelectorPlacement, selectedImageModel, setSelectedModel]);
}, [
modelSelectorPlacement,
selectedImageModel,
setSelectedModel,
stage3dModelList,
]);
const inlineModelItems: PopoverItem[] = useMemo(
() =>
STAGE_3D_PAGE_MODEL_LIST.map((item) => ({
stage3dModelList.map((item) => ({
...item,
selected: item.model === selectedImageModel,
})),
[selectedImageModel],
[selectedImageModel, stage3dModelList],
);
const handleInlineModelSelect = useCallback(
(item: PopoverItem) => {
@@ -188,9 +194,9 @@ export const Stage3DBody = ({
);
const selectedModelIcon = useMemo(
() =>
STAGE_3D_PAGE_MODEL_LIST.find((i) => i.model === selectedImageModel)
stage3dModelList.find((i) => i.model === selectedImageModel)
?.icon,
[selectedImageModel],
[selectedImageModel, stage3dModelList],
);
const inlineModelSelector =
modelSelectorPlacement === "prompt-box" ? (
@@ -536,7 +542,7 @@ export const Stage3DBody = ({
{modelSelectorPlacement === "bottom-left" && (
<div className="absolute bottom-6 left-6 z-20 flex items-center gap-3">
<ClassyModelSelector
items={STAGE_3D_PAGE_MODEL_LIST}
items={stage3dModelList}
page={PAGE_ID}
panelTitle="Select Model"
panelClassName="min-w-[300px]"
@@ -12,13 +12,13 @@ import {
useSelectedModel,
useSelectedProviderForModel,
defaultModelForPage,
TEXT_TO_IMAGE_PAGE_MODEL_LIST,
IMAGE_TO_VIDEO_PAGE_MODEL_LIST,
CANVAS_2D_PAGE_MODEL_LIST,
STAGE_3D_PAGE_MODEL_LIST,
IMAGE_EDITOR_PAGE_MODEL_LIST,
useTextToImagePageModelList,
useImageToVideoPageModelList,
useCanvas2dPageModelList,
useStage3dPageModelList,
useImageEditorPageModelList,
useAnglesPageModelList,
IMAGE_TO_3D_WORLD_PAGE_MODEL_LIST,
ANGLES_PAGE_MODEL_LIST,
} from "@storyteller/ui-model-selector";
import {
usePrompt2DStore,
@@ -49,41 +49,11 @@ const PROVIDER_DISPLAY_NAMES: Record<string, string> = {
worldlabs: "World Labs",
};
// Get models list for a page
const getModelsForPage = (page: ModelPage | null): Model[] => {
switch (page) {
case ModelPage.TextToImage:
return TEXT_TO_IMAGE_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
case ModelPage.ImageToVideo:
return IMAGE_TO_VIDEO_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
case ModelPage.Canvas2D:
return CANVAS_2D_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
case ModelPage.Stage3D:
return STAGE_3D_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
case ModelPage.ImageEditor:
return IMAGE_EDITOR_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
case ModelPage.ImageTo3DWorld:
return IMAGE_TO_3D_WORLD_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
case ModelPage.Angles:
return ANGLES_PAGE_MODEL_LIST.map((item) => item.model).filter(
(m): m is Model => m !== undefined,
);
default:
return [];
}
};
// Extract the concrete `Model` instances out of a page's selectable item list.
const modelsFromList = (
list: { model?: Model }[],
): Model[] =>
list.map((item) => item.model).filter((m): m is Model => m !== undefined);
export interface CostBreakdownModalProps {
/** The current active tab ID from the app (e.g. "IMAGE", "VIDEO", "2D", "3D") */
@@ -106,11 +76,47 @@ export function CostBreakdownModal({ activeTabId }: CostBreakdownModalProps) {
return TAB_TO_MODEL_PAGE[activeTabId] ?? ModelPage.TextToImage;
}, [activeTabId]);
// Backend-reconciled model lists per page (fall back to the overlay until the
// startup fetch completes).
const textToImageList = useTextToImagePageModelList();
const imageToVideoList = useImageToVideoPageModelList();
const canvas2dList = useCanvas2dPageModelList();
const stage3dList = useStage3dPageModelList();
const imageEditorList = useImageEditorPageModelList();
const anglesList = useAnglesPageModelList();
// Get the selected model for the active page
const selectedModelFromStore = useSelectedModel(activePage);
// If no model selected in store, use the default for this page
const modelsForPage = getModelsForPage(activePage);
const modelsForPage = useMemo<Model[]>(() => {
switch (activePage) {
case ModelPage.TextToImage:
return modelsFromList(textToImageList);
case ModelPage.ImageToVideo:
return modelsFromList(imageToVideoList);
case ModelPage.Canvas2D:
return modelsFromList(canvas2dList);
case ModelPage.Stage3D:
return modelsFromList(stage3dList);
case ModelPage.ImageEditor:
return modelsFromList(imageEditorList);
case ModelPage.ImageTo3DWorld:
return modelsFromList(IMAGE_TO_3D_WORLD_PAGE_MODEL_LIST);
case ModelPage.Angles:
return modelsFromList(anglesList);
default:
return [];
}
}, [
activePage,
textToImageList,
imageToVideoList,
canvas2dList,
stage3dList,
imageEditorList,
anglesList,
]);
const selectedModel =
selectedModelFromStore ?? defaultModelForPage(modelsForPage, activePage);
+2 -2
View File
@@ -22,5 +22,5 @@ export * from "./lib/classes/SplatModel.js";
export * from "./lib/lists/SplatModels.js";
export * from "./lib/classes/Object3DModel.js";
export * from "./lib/lists/Object3DModels.js";
export * from "./lib/loader/modelReconciliation.js";
export * from "./lib/loader/buildModelsFromListing.js";
@@ -0,0 +1,160 @@
// Builds the picker model lists FROM the backend omni listing (the Tauri
// command response), mirroring how the web apps render `res.models`. Membership
// and order come from the backend; the frontend overlay only supplies richer UI
// metadata (selectorName, badges, tags, size icons) when it has an entry for a
// model. Backend models with no overlay entry still appear via a minimal model
// built straight from the response, so NEW backend models show up immediately.
//
// Pure module (no tauri-api import). The store in `@storyteller/tauri-api` feeds
// it the command payload's `models[]`; the DTO is accepted structurally.
import { ImageModel } from "../classes/ImageModel.js";
import { VideoModel } from "../classes/VideoModel.js";
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
import {
BACKEND_TO_TAURI_IMAGE_ID,
BACKEND_TO_TAURI_VIDEO_ID,
modelCreatorFromBackend,
} from "./modelReconciliation.js";
// ── Structural shapes of the backend listing (subset we read) ──────────────
export interface ListingModelBase {
model: string;
model_creator?: string | null;
full_name?: string | null;
is_disabled?: boolean | null;
}
export interface ListingImageModel extends ListingModelBase {
text_prompt_supported?: boolean | null;
image_refs_supported?: boolean | null;
image_refs_max?: number | null;
}
export interface ListingVideoModel extends ListingModelBase {
extra_info_short?: string | null;
text_to_video_supported?: boolean | null;
starting_keyframe_supported?: boolean | null;
starting_keyframe_required?: boolean | null;
ending_keyframe_supported?: boolean | null;
resolution_options?: string[] | null;
resolution_default?: string | null;
duration_seconds_options?: number[] | null;
duration_seconds_default?: number | null;
show_generate_with_sound_toggle?: boolean | null;
}
// ── Public builders ────────────────────────────────────────────────────────
export const buildImageModelsFromListing = (
overlay: ImageModel[],
listing: ListingImageModel[],
): ImageModel[] =>
build(overlay, listing, BACKEND_TO_TAURI_IMAGE_ID, minimalImageModel);
export const buildVideoModelsFromListing = (
overlay: VideoModel[],
listing: ListingVideoModel[],
): VideoModel[] =>
build(overlay, listing, BACKEND_TO_TAURI_VIDEO_ID, minimalVideoModel);
// ── Core assembly ──────────────────────────────────────────────────────────
const build = <T extends { tauriId: string }, L extends ListingModelBase>(
overlay: T[],
listing: L[],
alias: Record<string, string>,
makeMinimal: (m: L, tauriId: string) => T,
): T[] => {
const overlayByTauriId = new Map(overlay.map((m) => [m.tauriId, m]));
const knownTauriIds = new Set(listing.map((m) => alias[m.model] ?? m.model));
const result: T[] = [];
// Backend drives membership + order.
for (const m of listing) {
if (m.is_disabled === true) continue;
const tauriId = alias[m.model] ?? m.model;
result.push(overlayByTauriId.get(tauriId) ?? makeMinimal(m, tauriId));
}
// Append frontend-only models the backend has never heard of (switch_x,
// inpaint, angle models, …).
for (const m of overlay) {
if (!knownTauriIds.has(m.tauriId)) result.push(m);
}
return result;
};
// ── Minimal models (backend model with no overlay entry) ───────────────────
const minimalImageModel = (m: ListingImageModel, tauriId: string): ImageModel =>
new ImageModel({
id: tauriId,
tauriId,
fullName: displayName(m),
category: "image",
creator: creatorFor(m.model_creator, m.model),
selectorName: displayName(m),
selectorDescription: "",
selectorBadges: [],
maxGenerationCount: 4,
defaultGenerationCount: 1,
canTextToImage: m.text_prompt_supported !== false,
canUseImagePrompt: m.image_refs_supported === true,
maxImagePromptCount: m.image_refs_max ?? 1,
});
const minimalVideoModel = (m: ListingVideoModel, tauriId: string): VideoModel =>
new VideoModel({
id: tauriId,
tauriId,
fullName: displayName(m),
category: "video",
creator: creatorFor(m.model_creator, m.model),
selectorName: displayName(m),
selectorDescription: m.extra_info_short ?? "",
selectorBadges: [],
startFrame: m.starting_keyframe_supported === true,
endFrame: m.ending_keyframe_supported === true,
requiresImage:
m.starting_keyframe_required === true || m.text_to_video_supported === false,
textToVideoSupported: m.text_to_video_supported !== false,
resolutionOptions: m.resolution_options ?? undefined,
defaultResolution: m.resolution_default ?? undefined,
durationOptions: m.duration_seconds_options ?? undefined,
defaultDuration: m.duration_seconds_default ?? undefined,
generateWithSound: m.show_generate_with_sound_toggle === true,
});
// ── Helpers ────────────────────────────────────────────────────────────────
const displayName = (m: ListingModelBase): string => m.full_name || m.model;
// Guess a creator from the `model_creator` field, falling back to the model-id
// prefix, then to ArtCraft.
const CREATOR_BY_PREFIX: Array<[string, ModelCreator]> = [
["flux", ModelCreator.BlackForestLabs],
["nano_banana", ModelCreator.Google],
["gpt_image", ModelCreator.OpenAi],
["seedream", ModelCreator.Bytedance],
["seedance", ModelCreator.Bytedance],
["kling", ModelCreator.Kling],
["sora", ModelCreator.OpenAi],
["veo", ModelCreator.Google],
["grok", ModelCreator.Grok],
["happy_horse", ModelCreator.Alibaba],
["midjourney", ModelCreator.Midjourney],
["qwen", ModelCreator.Alibaba],
];
const creatorFor = (
raw: string | null | undefined,
modelId: string,
): ModelCreator => {
const mapped = modelCreatorFromBackend(raw ?? undefined);
if (mapped) return mapped;
for (const [prefix, creator] of CREATOR_BY_PREFIX) {
if (modelId.startsWith(prefix)) return creator;
}
return ModelCreator.ArtCraft;
};
@@ -0,0 +1,106 @@
// Reconciles the frontend model OVERLAY (the hand-maintained `IMAGE_MODELS` /
// `VIDEO_MODELS`, which carry all the UI/product metadata + page-capability flags)
// against the backend omni listing (`ListImageModels` / `ListVideoModels`).
//
// The backend is authoritative for WHICH models are offered; the overlay is
// authoritative for the metadata the backend doesn't have (selectorName, badges,
// tags, canTextToImage/canEditImages/canEditAngles, sizeOptions, durations, and
// the frontend-only models switch_x / inpaint / angle / kontext).
//
// This module is intentionally PURE (no tauri-api import) so `model-list` stays
// free of a circular dependency. The async loader that calls the Tauri commands
// lives in `tauri-api` and feeds `knownBackendIds` / `offeredBackendIds` here.
import { ImageModel } from "../classes/ImageModel.js";
import { VideoModel } from "../classes/VideoModel.js";
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
/**
* Backend omni model id (the `CommonImageModel` serde string) -> frontend `tauriId`.
* Only the MISMATCHES need entries; matching ids pass through unchanged.
* See external/reports/2026-07/storyteller-web-omni-vs-tauri-model-differences.md.
*/
export const BACKEND_TO_TAURI_IMAGE_ID: Record<string, string> = {
flux_pro_1p1: "flux_pro_11",
flux_pro_1p1_ultra: "flux_pro_11_ultra",
grok_imagine_image: "grok_image",
// The backend has three versioned Midjourney models; the frontend has one "midjourney".
midjourney_7: "midjourney",
midjourney_7_niji: "midjourney",
midjourney_8: "midjourney",
};
export const BACKEND_TO_TAURI_VIDEO_ID: Record<string, string> = {
kling_1p6_pro: "kling_1.6_pro",
kling_2p1_pro: "kling_2.1_pro",
kling_2p1_master: "kling_2.1_master",
seedance_1p0_lite: "seedance_1.0_lite",
grok_imagine_video: "grok_video",
};
/** Backend `model_creator` snake_case string -> frontend `ModelCreator` enum. */
export const modelCreatorFromBackend = (raw?: string): ModelCreator | undefined => {
switch (raw) {
case "alibaba": return ModelCreator.Alibaba;
case "artcraft": return ModelCreator.ArtCraft;
case "black_forest_labs": return ModelCreator.BlackForestLabs;
case "bytedance": return ModelCreator.Bytedance;
case "fal": return ModelCreator.Fal;
case "google": return ModelCreator.Google;
case "grok": return ModelCreator.Grok;
case "hailuo": return ModelCreator.Hailuo;
case "higgsfield": return ModelCreator.Higgsfield;
case "kling": return ModelCreator.Kling;
case "krea": return ModelCreator.Krea;
case "midjourney": return ModelCreator.Midjourney;
case "open_ai": return ModelCreator.OpenAi;
case "open_art": return ModelCreator.OpenArt;
case "recraft": return ModelCreator.Recraft;
case "replicate": return ModelCreator.Replicate;
case "runway": return ModelCreator.Runway;
case "stability": return ModelCreator.Stability;
case "tencent": return ModelCreator.Tencent;
case "tensor_art": return ModelCreator.TensorArt;
case "vidu": return ModelCreator.Vidu;
case "world_labs": return ModelCreator.WorldLabs;
default: return undefined;
}
};
const toTauriIds = (backendIds: string[], alias: Record<string, string>): Set<string> =>
new Set(backendIds.map((id) => alias[id] ?? id));
/**
* Membership is BACKEND-DRIVEN, mirroring the web apps' `omni-gen-hooks` (which
* render `res.models.filter((m) => m.is_disabled !== true)`):
*
* - Keep an overlay model when the backend returns it ENABLED.
* - Drop it when the backend KNOWS the model but has it disabled (`is_disabled`).
* - Keep it when the backend has never heard of it — a frontend-only model like
* `switch_x` / inpaint / angles that has no omni counterpart.
*
* The overlay only supplies the UI metadata the backend lacks (selectorName,
* badges, tags, capability flags); the backend response decides what appears.
*/
const reconcile = <T extends { tauriId: string }>(
overlay: T[],
knownBackendIds: string[],
enabledBackendIds: string[],
alias: Record<string, string>,
): T[] => {
const known = toTauriIds(knownBackendIds, alias);
const enabled = toTauriIds(enabledBackendIds, alias);
return overlay.filter((m) => enabled.has(m.tauriId) || !known.has(m.tauriId));
};
export const reconcileImageModels = (
overlay: ImageModel[],
knownBackendIds: string[],
enabledBackendIds: string[],
): ImageModel[] => reconcile(overlay, knownBackendIds, enabledBackendIds, BACKEND_TO_TAURI_IMAGE_ID);
export const reconcileVideoModels = (
overlay: VideoModel[],
knownBackendIds: string[],
enabledBackendIds: string[],
): VideoModel[] => reconcile(overlay, knownBackendIds, enabledBackendIds, BACKEND_TO_TAURI_VIDEO_ID);
+4 -1
View File
@@ -16,6 +16,8 @@ export * from "./lib/enqueue/EnqueueImageToVideo.js";
export * from "./lib/enqueue/EnqueueTextToImage.js";
export * from "./lib/generate/GenerateImage.js";
export * from "./lib/generate/GenerateVideo.js";
export * from "./lib/generate/models/image/ListImageModels.js";
export * from "./lib/generate/models/video/ListVideoModels.js";
export * from "./lib/events/BasicEventWrapper.js";
export * from "./lib/events/CanvasBgRemovedEvent.js";
export * from "./lib/fal/FalBackgroundRemoval.js";
@@ -45,4 +47,5 @@ export * from "./lib/util/LoadWithoutCors.js";
export * from "./lib/worldlabs/WorldLabsGetCredentialInfo.js";
export * from "./lib/cost_estimate/EstimateImageCost.js";
export * from "./lib/cost_estimate/EstimateSplatCost.js";
export * from "./lib/cost_estimate/EstimateVideoCost.js";
export * from "./lib/cost_estimate/EstimateVideoCost.js";
export * from "./lib/models/modelsStore.js";
@@ -0,0 +1,63 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../../../common/CommandStatus";
// NB: enum-valued fields are typed as plain strings on purpose. The backend
// enums carry an `Unknown(String)` catch-all, so a newer server may return a
// value this build doesn't know; keeping them as strings keeps us forward-compatible.
export interface ListImageModelsSuccess extends CommandResult {
payload: ListImageModelsPayload;
}
export interface ListImageModelsPayload {
success: boolean;
models: OmniGenImageModelDetails[];
providers: OmniGenImageModelProviderDetails[];
}
export interface OmniGenImageModelProviderDetails {
provider: string;
models: OmniGenImageProviderModelDetails[];
}
export interface OmniGenImageProviderModelDetails {
model: string;
overrides?: OmniGenImageModelDetails;
}
export interface OmniGenImageModelDetails {
model: string;
model_creator?: string;
full_name?: string;
text_prompt_supported?: boolean;
text_prompt_max_length?: number;
negative_text_prompt_supported?: boolean;
negative_text_prompt_max_length?: number;
image_refs_supported?: boolean;
image_refs_max?: number;
has_fixed_editing_aspect_ratio?: boolean;
aspect_ratio_options?: string[];
aspect_ratio_default?: string;
aspect_ratio_default_when_editing?: string;
resolution_options?: string[];
resolution_default?: string;
quality_options?: string[];
default_quality?: string;
batch_size_min?: number;
batch_size_max?: number;
batch_size_options?: number[];
batch_size_default?: number;
is_disabled?: boolean;
}
/**
* List available image models from the Tauri backend
* (`list_image_models_command` — retried + cached 60s in the Rust layer).
*/
export const ListImageModels = async (): Promise<ListImageModelsSuccess> => {
try {
return (await invoke("list_image_models_command")) as ListImageModelsSuccess;
} catch (error) {
throw error;
}
};
@@ -0,0 +1,82 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../../../common/CommandStatus";
// NB: enum-valued fields are typed as plain strings for forward-compatibility
// (the backend enums have an `Unknown(String)` catch-all).
export interface ListVideoModelsSuccess extends CommandResult {
payload: ListVideoModelsPayload;
}
export interface ListVideoModelsPayload {
success: boolean;
models: OmniGenVideoModelDetails[];
providers: OmniGenVideoModelProviderDetails[];
}
export interface OmniGenVideoModelProviderDetails {
provider: string;
models: OmniGenVideoProviderModelDetails[];
}
export interface OmniGenVideoProviderModelDetails {
model: string;
overrides?: OmniGenVideoModelDetails;
}
export interface OmniGenVideoModelDetails {
model: string;
model_creator?: string;
full_name?: string;
extra_info?: string;
extra_info_short?: string;
text_to_video_supported?: boolean;
text_prompt_supported?: boolean;
text_prompt_max_length?: number;
negative_text_prompt_supported?: boolean;
negative_text_prompt_max_length?: number;
starting_keyframe_supported?: boolean;
starting_keyframe_required?: boolean;
ending_keyframe_supported?: boolean;
image_references_supported?: boolean;
image_references_max?: number;
video_references_supported?: boolean;
video_references_max?: number;
video_references_max_total_duration_seconds?: number;
audio_references_supported?: boolean;
audio_references_max?: number;
audio_references_max_total_duration_seconds?: number;
character_references_supported?: boolean;
character_references_max?: number;
show_generate_with_sound_toggle?: boolean;
aspect_ratio_options?: string[];
aspect_ratio_default?: string;
resolution_options?: string[];
resolution_default?: string;
bitrate_options?: string[];
bitrate_default?: string;
quality_options?: string[];
default_quality?: string;
duration_seconds_min?: number;
duration_seconds_max?: number;
duration_seconds_max_with_image_references?: number;
duration_seconds_options?: number[];
duration_seconds_default?: number;
batch_size_min?: number;
batch_size_max?: number;
batch_size_options?: number[];
batch_size_default?: number;
is_disabled?: boolean;
}
/**
* List available video models from the Tauri backend
* (`list_video_models_command` — retried + cached 60s in the Rust layer).
*/
export const ListVideoModels = async (): Promise<ListVideoModelsSuccess> => {
try {
return (await invoke("list_video_models_command")) as ListVideoModelsSuccess;
} catch (error) {
throw error;
}
};
@@ -0,0 +1,79 @@
// Shared model store: the single source of truth for the model dropdowns.
//
// Seeded synchronously from the frontend OVERLAY (`IMAGE_MODELS` / `VIDEO_MODELS`
// in `@storyteller/model-list`) so the UI is never empty. On app boot,
// `loadModelsFromBackend()` fetches the authoritative omni listing via the Tauri
// commands and rebuilds the lists FROM that response (membership + order come
// from the backend; the overlay only enriches UI metadata; backend models with
// no overlay entry are built minimally so NEW models appear). If the fetch fails
// the store keeps the overlay.
import { create } from "zustand";
import {
ImageModel,
VideoModel,
IMAGE_MODELS,
VIDEO_MODELS,
buildImageModelsFromListing,
buildVideoModelsFromListing,
} from "@storyteller/model-list";
import { ListImageModels } from "../generate/models/image/ListImageModels.js";
import { ListVideoModels } from "../generate/models/video/ListVideoModels.js";
export interface ModelsStoreState {
imageModels: ImageModel[];
videoModels: VideoModel[];
// True once a backend reconciliation has completed at least once.
loaded: boolean;
isLoading: boolean;
loadModelsFromBackend: () => Promise<void>;
}
export const useModelsStore = create<ModelsStoreState>((set, get) => ({
imageModels: IMAGE_MODELS,
videoModels: VIDEO_MODELS,
loaded: false,
isLoading: false,
loadModelsFromBackend: async () => {
if (get().isLoading) return;
console.log("[models] loadModelsFromBackend() — calling Tauri commands…");
set({ isLoading: true });
const [imageResult, videoResult] = await Promise.allSettled([
ListImageModels(),
ListVideoModels(),
]);
const next: Partial<ModelsStoreState> = { isLoading: false, loaded: true };
if (imageResult.status === "fulfilled") {
const { models } = imageResult.value.payload;
const built = buildImageModelsFromListing(IMAGE_MODELS, models);
next.imageModels = built;
console.log(
`[models] image: backend returned ${models.length} → ${built.length} models shown`,
built.map((m) => m.tauriId),
);
} else {
console.error("[models] failed to load image models from backend:", imageResult.reason);
}
if (videoResult.status === "fulfilled") {
const { models } = videoResult.value.payload;
const built = buildVideoModelsFromListing(VIDEO_MODELS, models);
next.videoModels = built;
console.log(
`[models] video: backend returned ${models.length} → ${built.length} models shown`,
built.map((m) => m.tauriId),
);
} else {
console.error("[models] failed to load video models from backend:", videoResult.reason);
}
set(next);
},
}));
// Hook selectors for React consumers.
export const useImageModels = (): ImageModel[] => useModelsStore((s) => s.imageModels);
export const useVideoModels = (): VideoModel[] => useModelsStore((s) => s.videoModels);