Webapp: move audio/3D reference inputs into the prompt box reference deck (#1795)

## Summary

Compacts the legacy reference bands above the prompt box (audio ref row,
multi-view/input-mesh row, reference video row) into the reference deck
design already used on the image and video pages. Mobile forms keep
their existing rows.

## Changes

- **Create Audio** - audio tracks are now deck cards (playable tile,
duration badge) with Upload / From library in the deck's "+" menu; band
removed. Mutual-exclusion and "Audio track required" behavior unchanged.
- **Create 3D World** - guide video is now a deck card; store reworked
to the shared `RefVideo[]` shape.
- **Create 3D Object** - Front/Back/Left/Right are always-visible
labeled slot cards next to the image-ref deck (same design as First/Last
frame on video), plus a "Mesh" slot card for mesh-to-mesh models. Upload
logic deduplicated into `mesh-upload.ts`.
- **Webapp PromptBox** - removed the `mediaReferenceRow` band prop; deck
video/audio groups gate on `videoRefsSupported`/`audioRefsSupported` (no
longer video-page-only); new `referenceSlots` prop; `maxImages: 0`
blocks image uploads on pages whose model takes no image refs.
- **Deck lib** - exported `DeckSlotCard`; new `mesh` card kind;
`sortable: false` for fixed slots; non-finite duration caps hide the
seconds limit; collapsed fan width now tracks card count, so a single
ref sits snug against the prompt (multi-ref unchanged).
This commit is contained in:
BFlat authored and GitHub committed 2026-07-20 21:12:01 +07:00
1 parent 185bf835a1
commit eb59e7fcd7
17 files changed
+510 -222

No files matched your search

@@ -213,7 +213,7 @@ export const MediaReferenceRow = ({
/>
<div
className={twMerge(
"glass flex flex-col sm:flex-row rounded-2xl sm:rounded-none",
"glass flex flex-col sm:flex-row rounded-2xl",
className,
)}
onMouseDown={(e) => e.stopPropagation()}
@@ -81,7 +81,8 @@ interface PromptBoxProps {
onClearAllExtras?: () => void;
hasClearableExtras?: boolean;
// Reference-mode media (video page passes these; other pages omit)
// Video/audio references, rendered as cards in the reference deck (video
// page in reference mode, world page's guide video, audio page's tracks).
videoRefsSupported?: boolean;
referenceVideos?: RefVideo[];
onReferenceVideosChange?: (videos: RefVideo[]) => void;
@@ -93,9 +94,9 @@ interface PromptBoxProps {
maxAudioCount?: number;
maxAudioRefDuration?: number;
// Legacy band rendered above the box (audio/object/world pages still use
// this; the image/video pages moved to the inline reference deck).
mediaReferenceRow?: ReactNode;
// Always-visible named slot cards rendered beside the reference deck
// (object page's multi-view angles + input mesh), built from DeckSlotCard.
referenceSlots?: ReactNode;
// Model selector (rendered above the toolbar, typically hidden on desktop via lg:hidden)
modelSelector?: ReactNode;
@@ -151,7 +152,7 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
onReferenceAudiosChange,
maxAudioCount = 2,
maxAudioRefDuration = 30,
mediaReferenceRow,
referenceSlots,
modelSelector,
secondaryPromptRow,
mentionItems,
@@ -195,7 +196,9 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
const deck = useDeckMedia({
referenceImages,
setReferenceImages: onReferenceImagesChange,
maxImages: maxImagePromptCount,
// 0 blocks image uploads (incl. via the combined picker) on pages whose
// model only takes audio/video/mesh refs.
maxImages: supportsImagePrompts ? maxImagePromptCount : 0,
setEndFrameImage: onEndFrameImageChange,
referenceVideos,
setReferenceVideos: onReferenceVideosChange,
@@ -278,8 +281,9 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
const deckAddActions: DeckAddAction[] = [];
if (
supportsImagePrompts &&
referenceImages.length + deck.uploadingImages.length <
maxImagePromptCount
maxImagePromptCount
) {
deckAddActions.push({
key: "upload-image",
@@ -297,8 +301,6 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
}
}
if (
isVideo &&
isReferenceMode &&
videoRefsSupported &&
referenceVideos.length < maxVideoCount &&
!deck.uploadingVideo
@@ -319,8 +321,6 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
}
}
if (
isVideo &&
isReferenceMode &&
audioRefsSupported &&
referenceAudios.length < maxAudioCount &&
!deck.uploadingAudio
@@ -422,10 +422,10 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
};
// Left-of-textarea reference widget: image deck, keyframe cards, or the
// mixed reference-mode deck depending on page/mode.
// mixed deck depending on page/mode.
const renderReferenceWidget = (alwaysExpanded?: boolean) => {
if (!supportsImagePrompts) return null;
if (isVideo && !isReferenceMode) {
if (!supportsImagePrompts) return null;
return (
<KeyframeCards
firstFrame={firstFrameItem}
@@ -469,6 +469,9 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
/>
);
}
if (!supportsImagePrompts && !videoRefsSupported && !audioRefsSupported) {
return null;
}
const totalVideoRefSeconds = referenceVideos.reduce(
(sum, video) => sum + video.duration,
0,
@@ -477,16 +480,24 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
(sum, audio) => sum + audio.duration,
0,
);
const groupHints: Record<string, string> = {
image: `${referenceImages.length}/${maxImagePromptCount}`,
};
if (isVideo && isReferenceMode) {
if (videoRefsSupported) {
groupHints.video = `${referenceVideos.length}/${maxVideoCount} · ${totalVideoRefSeconds}/${maxVideoRefDuration}s`;
}
if (audioRefsSupported) {
groupHints.audio = `${referenceAudios.length}/${maxAudioCount} · ${totalAudioRefSeconds}/${maxAudioRefDuration}s`;
}
const groupHints: Record<string, string> = {};
if (supportsImagePrompts) {
groupHints.image = `${referenceImages.length}/${maxImagePromptCount}`;
}
// A non-finite duration cap means "no limit" — show counts only.
if (videoRefsSupported) {
groupHints.video =
`${referenceVideos.length}/${maxVideoCount}` +
(isFinite(maxVideoRefDuration)
? ` · ${totalVideoRefSeconds}/${maxVideoRefDuration}s`
: "");
}
if (audioRefsSupported) {
groupHints.audio =
`${referenceAudios.length}/${maxAudioCount}` +
(isFinite(maxAudioRefDuration)
? ` · ${totalAudioRefSeconds}/${maxAudioRefDuration}s`
: "");
}
return (
@@ -495,7 +506,11 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
canAdd={deckAddActions.length > 0}
addActions={deckAddActions}
addMenuGroupHints={groupHints}
onAddClick={deck.openAnyUpload}
onAddClick={
supportsImagePrompts || videoRefsSupported || audioRefsSupported
? deck.openAnyUpload
: undefined
}
onRemove={handleRemoveDeckItem}
onReorderImages={(from, to) =>
onReferenceImagesChange(arrayMove(referenceImages, from, to))
@@ -690,19 +705,15 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
<div className="relative flex flex-col">
{deck.fileInputs}
{mediaReferenceRow && (
<div className="sm:rounded-t-2xl">{mediaReferenceRow}</div>
)}
<div
className={twMerge(
"glass rounded-2xl p-3 sm:p-4 !transition-all duration-200",
mediaReferenceRow && "rounded-t-none border-t-0",
isFocused && "ring-1 ring-primary",
)}
>
<div className="flex gap-3">
{renderReferenceWidget()}
{referenceSlots}
<div className="promptbox-resize-wrap relative flex-1">
{hasMentionItems && mentionItems ? (
@@ -908,7 +919,14 @@ export const PromptBox = forwardRef<HTMLDivElement, PromptBoxProps>(
{leftToolbar}
</>
}
imagePromptRow={renderReferenceWidget(true) ?? undefined}
imagePromptRow={
renderReferenceWidget(true) || referenceSlots ? (
<div className="flex flex-wrap items-center gap-3">
{renderReferenceWidget(true)}
{referenceSlots}
</div>
) : undefined
}
clearAllButton={
<PromptClearAllButton
onClick={handleClearAll}
@@ -121,22 +121,17 @@ export default function CreateAudio() {
const ui = useCreateAudioStore((s) => s.ui);
const setUi = useCreateAudioStore((s) => s.setUi);
const selectedModel = useMemo(
(): OmniGenAudioModelDetails | undefined => {
if (!apiModels.length) return undefined;
if (ui.selectedModelId) {
return (
apiModels.find((m) => m.model === ui.selectedModelId) ??
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ??
apiModels[0]
);
}
const selectedModel = useMemo((): OmniGenAudioModelDetails | undefined => {
if (!apiModels.length) return undefined;
if (ui.selectedModelId) {
return (
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0]
apiModels.find((m) => m.model === ui.selectedModelId) ??
apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ??
apiModels[0]
);
},
[apiModels, ui.selectedModelId],
);
}
return apiModels.find((m) => m.model === DEFAULT_MODEL_ID) ?? apiModels[0];
}, [apiModels, ui.selectedModelId]);
const prompt = ui.prompt;
const setPrompt = useCallback((v: string) => setUi({ prompt: v }), [setUi]);
@@ -287,7 +282,9 @@ export default function CreateAudio() {
(audios: typeof referenceAudios) => {
if (audios.length > 0 && referenceImages.length > 0) {
setReferenceImages([]);
toast.error("Removed image reference — it can't be combined with audio");
toast.error(
"Removed image reference — it can't be combined with audio",
);
}
setReferenceAudios(audios);
},
@@ -298,7 +295,9 @@ export default function CreateAudio() {
(images: RefImage[]) => {
if (images.length > 0 && referenceAudios.length > 0) {
setReferenceAudios([]);
toast.error("Removed audio reference — it can't be combined with an image");
toast.error(
"Removed audio reference — it can't be combined with an image",
);
}
setReferenceImages(images);
},
@@ -363,13 +362,15 @@ export default function CreateAudio() {
const handleLibraryImageSelect = useCallback(
(items: GalleryItem[]) => {
const newImages: RefImage[] = items.slice(0, maxImageRefs).map((item) => ({
id: Math.random().toString(36).substring(7),
url: item.thumbnail || item.fullImage || "",
fullUrl: item.fullImage || undefined,
file: new File([], "library-image"),
mediaToken: item.id,
}));
const newImages: RefImage[] = items
.slice(0, maxImageRefs)
.map((item) => ({
id: Math.random().toString(36).substring(7),
url: item.thumbnail || item.fullImage || "",
fullUrl: item.fullImage || undefined,
file: new File([], "library-image"),
mediaToken: item.id,
}));
handleReferenceImagesChange(newImages);
setIsImagePickerOpen(false);
},
@@ -547,6 +548,7 @@ export default function CreateAudio() {
</>
);
// Mobile-only band; on desktop the audio refs live in the reference deck.
const audioReferenceRow = audioRefsSupported ? (
<MediaReferenceRow
videoSupported={false}
@@ -787,7 +789,7 @@ export default function CreateAudio() {
promptBox={
<div
ref={promptBoxRef}
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-6xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
style={{
animationDelay: "150ms",
left: "var(--ac-sidebar-offset, 0px)",
@@ -806,14 +808,14 @@ export default function CreateAudio() {
referenceImages={referenceImages}
onReferenceImagesChange={handleReferenceImagesChange}
onPickFromLibrary={() => setIsImagePickerOpen(true)}
onClearAllExtras={() => {
setStylePrompt("");
setReferenceAudios([]);
}}
hasClearableExtras={
ui.stylePrompt.length > 0 || referenceAudios.length > 0
}
mediaReferenceRow={audioReferenceRow}
audioRefsSupported={audioRefsSupported}
referenceAudios={referenceAudios}
onReferenceAudiosChange={handleReferenceAudiosChange}
maxAudioCount={maxAudioRefs}
maxAudioRefDuration={AUDIO_REF_MAX_DURATION_SECONDS}
onPickAudioFromLibrary={() => setIsAudioPickerOpen(true)}
onClearAllExtras={() => setStylePrompt("")}
hasClearableExtras={ui.stylePrompt.length > 0}
secondaryPromptRow={
styleSupported ? (
<StylePromptRow
@@ -617,7 +617,7 @@ export default function CreateImage() {
promptBox={
<div
ref={promptBoxRef}
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-6xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
style={{
animationDelay: "150ms",
left: "var(--ac-sidebar-offset, 0px)",
@@ -652,7 +652,9 @@ export default function CreateImage() {
richList
triggerIcon={
<img
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
src={getCreatorIconPathForModelId(
selectedModel?.model ?? "",
)}
alt=""
className="h-4 w-4 icon-auto-contrast"
/>
@@ -0,0 +1,170 @@
import { useRef, useState } from "react";
import {
DeckPreviewModal,
DeckSlotCard,
type DeckItem,
} from "@storyteller/ui-promptbox";
import type { MeshInputsState } from "../create-object-store";
import type { MultiViewSlot } from "./MeshInputsRow";
import { MESH_FILE_ACCEPT, uploadMeshFile, uploadViewImage } from "./mesh-upload";
// Always-visible named slot cards (Front/Back/Left/Right angles + input mesh)
// rendered beside the reference deck in the desktop prompt box — same design
// as the video page's First/Last frame cards. The mobile form keeps the
// MeshInputsRow band; both write to the same store slice.
const VIEW_SLOTS: {
slot: MultiViewSlot;
key: keyof MeshInputsState;
label: string;
}[] = [
{ slot: "front", key: "frontImage", label: "Front" },
{ slot: "back", key: "backImage", label: "Back" },
{ slot: "left", key: "leftImage", label: "Left" },
{ slot: "right", key: "rightImage", label: "Right" },
];
interface MeshDeckSlotsProps {
showMultiView: boolean;
showMeshInput: boolean;
inputs: MeshInputsState;
setInputs: (patch: Partial<MeshInputsState>) => void;
onPickSlotFromLibrary: (slot: MultiViewSlot) => void;
}
export function MeshDeckSlots({
showMultiView,
showMeshInput,
inputs,
setInputs,
onPickSlotFromLibrary,
}: MeshDeckSlotsProps) {
const viewInputRef = useRef<HTMLInputElement>(null);
const meshInputRef = useRef<HTMLInputElement>(null);
const targetSlotRef = useRef<MultiViewSlot>("front");
const [uploadingView, setUploadingView] = useState<{
slot: MultiViewSlot;
previewUrl: string;
} | null>(null);
const [isUploadingMesh, setIsUploadingMesh] = useState(false);
const [previewItem, setPreviewItem] = useState<DeckItem | null>(null);
const handleViewFile = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (viewInputRef.current) viewInputRef.current.value = "";
if (!file) return;
const target = VIEW_SLOTS.find((v) => v.slot === targetSlotRef.current)!;
const previewUrl = URL.createObjectURL(file);
setUploadingView({ slot: target.slot, previewUrl });
const image = await uploadViewImage(target.label, file);
URL.revokeObjectURL(previewUrl);
setUploadingView(null);
if (image) setInputs({ [target.key]: image });
};
const handleMeshFile = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (meshInputRef.current) meshInputRef.current.value = "";
if (!file) return;
setIsUploadingMesh(true);
const mesh = await uploadMeshFile(file);
setIsUploadingMesh(false);
if (mesh) setInputs({ inputMesh: mesh });
};
const meshItem: DeckItem | undefined = inputs.inputMesh
? {
id: inputs.inputMesh.id,
kind: "mesh",
name: inputs.inputMesh.file?.name || "Mesh file",
}
: isUploadingMesh
? { id: "uploading-mesh", kind: "mesh", name: "Mesh file", uploading: true }
: undefined;
return (
<div className="flex shrink-0 items-center gap-1.5 self-center">
<input
type="file"
ref={viewInputRef}
className="hidden"
accept="image/*"
onChange={handleViewFile}
/>
<input
type="file"
ref={meshInputRef}
className="hidden"
accept={MESH_FILE_ACCEPT}
onChange={handleMeshFile}
/>
{showMultiView &&
VIEW_SLOTS.map(({ slot, key, label }) => {
const image = inputs[key];
const item: DeckItem | undefined = image
? {
id: image.id,
kind: "image",
url: image.url,
previewUrl: image.fullUrl ?? image.url,
name: `${label} view`,
}
: uploadingView?.slot === slot
? {
id: `uploading-${slot}`,
kind: "image",
url: uploadingView.previewUrl,
name: `${label} view`,
uploading: true,
}
: undefined;
return (
<DeckSlotCard
key={slot}
item={item}
label={label}
addActions={[
{
key: `upload-${slot}`,
label: "Upload",
onSelect: () => {
targetSlotRef.current = slot;
viewInputRef.current?.click();
},
},
{
key: `library-${slot}`,
label: "From library",
onSelect: () => onPickSlotFromLibrary(slot),
},
]}
onRemove={() => setInputs({ [key]: undefined })}
onPreview={setPreviewItem}
/>
);
})}
{showMeshInput && (
<DeckSlotCard
item={meshItem}
label="Mesh"
addActions={[
{
key: "upload-mesh",
label: "Upload mesh",
onSelect: () => meshInputRef.current?.click(),
},
]}
onRemove={() => setInputs({ inputMesh: undefined })}
onPreview={setPreviewItem}
/>
)}
<DeckPreviewModal
item={previewItem}
onClose={() => setPreviewItem(null)}
/>
</div>
);
}
@@ -6,16 +6,12 @@ import {
faCube,
faImage,
} from "@fortawesome/pro-solid-svg-icons";
import { MediaUploadApi } from "@storyteller/api";
import { FilterEngineCategories } from "@storyteller/api";
import { UploaderStates } from "@storyteller/common";
import { uploadImage } from "../../../components/prompt-box/upload-image";
import { AddButton, type RefImage } from "../../../components/prompt-box";
import { MESH_FILE_ACCEPT, uploadMeshFile, uploadViewImage } from "./mesh-upload";
// Multi-view side inputs (front/back/left/right) and the mesh-to-mesh input
// file, shown above the prompt box when the selected model supports them. Uses
// the same media upload path (and Upload / Pick-from-library affordance) as the
// primary reference image row.
// file — the mobile form band. The desktop prompt box renders the same store
// slice as reference-deck cards via useMeshDeckRefs.
export type MultiViewSlot = "front" | "back" | "left" | "right";
@@ -38,8 +34,6 @@ interface MeshInputsRowProps {
onPickSlotFromLibrary?: (slot: MultiViewSlot) => void;
}
const MESH_FILE_ACCEPT = ".glb,.gltf,.fbx,.obj";
export function MeshInputsRow({
showMultiView,
showMeshInput,
@@ -63,9 +57,8 @@ export function MeshInputsRow({
return (
// Title on the left, upload slots right-aligned + top-aligned (matches
// ImagePromptRow). Flat bottom on desktop so it seams into the prompt box
// glass stacked below it; fully rounded on mobile (stands alone there).
<div className="glass flex items-start gap-3 rounded-2xl px-3 py-2 sm:rounded-t-2xl sm:rounded-b-none">
// ImagePromptRow).
<div className="glass flex items-start gap-3 rounded-2xl px-3 py-2">
<div className="flex grow flex-col gap-1 min-w-32">
<div className="flex items-center gap-2 text-white/90">
<FontAwesomeIcon icon={titleIcon} className="h-3.5 w-3.5" />
@@ -163,35 +156,14 @@ function ImageSlot({
const inputRef = useRef<HTMLInputElement>(null);
const [uploading, setUploading] = useState(false);
const handleUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
const handleUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (inputRef.current) inputRef.current.value = "";
if (!file) return;
setUploading(true);
const reader = new FileReader();
reader.onloadend = async () => {
await uploadImage({
title: `${label.toLowerCase()}-view-${Math.random().toString(36).substring(2, 10)}`,
assetFile: file,
progressCallback: (state) => {
if (state.status === UploaderStates.success && state.data) {
onChange({
id: Math.random().toString(36).substring(7),
url: reader.result as string,
file,
mediaToken: state.data,
});
setUploading(false);
} else if (
state.status === UploaderStates.assetError ||
state.status === UploaderStates.imageCreateError
) {
setUploading(false);
}
},
});
if (inputRef.current) inputRef.current.value = "";
};
reader.readAsDataURL(file);
const image = await uploadViewImage(label, file);
setUploading(false);
if (image) onChange(image);
};
return (
@@ -247,31 +219,12 @@ function MeshFileSlot({
const handleUpload = async (event: React.ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (inputRef.current) inputRef.current.value = "";
if (!file) return;
setUploading(true);
try {
const api = new MediaUploadApi();
const response = await api.UploadNewEngineAsset({
file,
fileName: file.name || `input-mesh-${Date.now()}`,
uuid: crypto.randomUUID(),
engine_category: FilterEngineCategories.OBJECT,
maybe_title: "input_mesh",
});
if (response?.success && response.data) {
onChange({
id: Math.random().toString(36).substring(7),
url: "",
file,
mediaToken: response.data,
});
}
} catch {
// ignore — user can retry
} finally {
setUploading(false);
if (inputRef.current) inputRef.current.value = "";
}
const mesh = await uploadMeshFile(file);
setUploading(false);
if (mesh) onChange(mesh);
};
return (
@@ -0,0 +1,63 @@
import { MediaUploadApi, FilterEngineCategories } from "@storyteller/api";
import { UploaderStates } from "@storyteller/common";
import { uploadImage } from "../../../components/prompt-box/upload-image";
import type { RefImage } from "../../../components/prompt-box";
// Upload paths shared by the desktop reference deck (useMeshDeckRefs) and the
// mobile MeshInputsRow band.
export const MESH_FILE_ACCEPT = ".glb,.gltf,.fbx,.obj";
const randomId = () => Math.random().toString(36).substring(7);
const readAsDataUrl = (file: File): Promise<string> =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => resolve(reader.result as string);
reader.onerror = reject;
reader.readAsDataURL(file);
});
/** Uploads a multi-view angle image; resolves null on failure. */
export async function uploadViewImage(
label: string,
file: File,
): Promise<RefImage | null> {
const url = await readAsDataUrl(file);
return new Promise((resolve) => {
uploadImage({
title: `${label.toLowerCase()}-view-${Math.random().toString(36).substring(2, 10)}`,
assetFile: file,
progressCallback: (state) => {
if (state.status === UploaderStates.success && state.data) {
resolve({ id: randomId(), url, file, mediaToken: state.data });
} else if (
state.status === UploaderStates.assetError ||
state.status === UploaderStates.imageCreateError
) {
resolve(null);
}
},
});
});
}
/** Uploads a mesh file via the engine-asset endpoint; resolves null on failure. */
export async function uploadMeshFile(file: File): Promise<RefImage | null> {
try {
const api = new MediaUploadApi();
const response = await api.UploadNewEngineAsset({
file,
fileName: file.name || `input-mesh-${Date.now()}`,
uuid: crypto.randomUUID(),
engine_category: FilterEngineCategories.OBJECT,
maybe_title: "input_mesh",
});
if (response?.success && response.data) {
return { id: randomId(), url: "", file, mediaToken: response.data };
}
} catch {
// fall through — user can retry
}
return null;
}
@@ -4,11 +4,7 @@ import { PopoverMenu, type PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { Button, ToggleButton } from "@storyteller/ui-button";
import { GalleryModal, type GalleryItem } from "@storyteller/ui-gallery-modal";
import {
faSparkles,
faGem,
faImage,
} from "@fortawesome/pro-solid-svg-icons";
import { faSparkles, faGem, faImage } from "@fortawesome/pro-solid-svg-icons";
import {
PromptBox,
ImagePromptRow,
@@ -37,6 +33,7 @@ import {
QUALITY_OPTIONS,
} from "./components/MeshOptionPickers";
import { MeshInputsRow, type MultiViewSlot } from "./components/MeshInputsRow";
import { MeshDeckSlots } from "./components/MeshDeckSlots";
import { useMeshCostEstimate } from "../../lib/cost-estimate-api";
import { resolveModelOption } from "../../lib/resolve-model-setting";
import {
@@ -124,8 +121,16 @@ export default function CreateObject() {
// Settings are sticky across model switches, resolved against the current
// model's supported options at read time (see lib/resolve-model-setting).
const meshOutputType = resolveModelOption(ui.meshOutputType, outputTypes, outputTypes[0]);
const polygonType = resolveModelOption(ui.polygonType, polygonTypes, polygonTypes[0]);
const meshOutputType = resolveModelOption(
ui.meshOutputType,
outputTypes,
outputTypes[0],
);
const polygonType = resolveModelOption(
ui.polygonType,
polygonTypes,
polygonTypes[0],
);
const geometryQuality = resolveModelOption(
ui.geometryQuality,
supportsGeometryQuality ? QUALITY_OPTIONS : [],
@@ -138,7 +143,9 @@ export default function CreateObject() {
);
// Toggles default on when supported unless the user turned them off.
const enablePbr = supportsPbr ? (ui.enablePbr ?? false) : false;
const enableTexture = supportsTextureToggle ? (ui.enableTexture ?? true) : true;
const enableTexture = supportsTextureToggle
? (ui.enableTexture ?? true)
: true;
const [isGenerating, setIsGenerating] = useState(false);
const referenceImages = useCreateObjectStore((s) => s.referenceImages);
@@ -458,12 +465,23 @@ export default function CreateObject() {
const placeholder = supportsText
? "Describe the 3D object you want..."
: supportsMeshInput
? "Upload a mesh file below to process"
: "Add a reference image below";
? "Upload a mesh file to process"
: "Add a reference image";
// Only render the extra-inputs row when the model actually supports one of its
// slots, otherwise it renders empty but still flags a "row above" the prompt
// box (flat-top visual glitch).
// Desktop: multi-view angles + input mesh render as always-visible slot
// cards beside the reference deck (like the video page's keyframe cards).
const meshDeckSlots =
supportsMultiView || supportsMeshInput ? (
<MeshDeckSlots
showMultiView={supportsMultiView}
showMeshInput={supportsMeshInput}
inputs={inputs}
setInputs={setInputs}
onPickSlotFromLibrary={setLibraryTarget}
/>
) : undefined;
// Mobile-only band over the same store slice.
const meshInputsRow =
supportsMultiView || supportsMeshInput ? (
<MeshInputsRow
@@ -619,7 +637,7 @@ export default function CreateObject() {
promptBox={
<div
ref={promptBoxRef}
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-6xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
style={{
animationDelay: "150ms",
left: "var(--ac-sidebar-offset, 0px)",
@@ -648,9 +666,14 @@ export default function CreateObject() {
})
}
hasClearableExtras={Object.values(inputs).some(Boolean)}
mediaReferenceRow={meshInputsRow}
referenceSlots={meshDeckSlots}
modelSelector={
<Tooltip content="Model" position="top" className="z-50" closeOnClick>
<Tooltip
content="Model"
position="top"
className="z-50"
closeOnClick
>
<PopoverMenu
items={modelItems}
onSelect={handleModelChange}
@@ -660,7 +683,9 @@ export default function CreateObject() {
richList
triggerIcon={
<img
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
src={getCreatorIconPathForModelId(
selectedModel?.model ?? "",
)}
alt=""
className="h-4 w-4 icon-auto-contrast"
/>
@@ -1491,7 +1491,7 @@ export default function CreateVideo() {
promptBox={
<div
ref={promptBoxRef}
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-6xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
style={{
animationDelay: "150ms",
left: "var(--ac-sidebar-offset, 0px)",
@@ -6,11 +6,15 @@ import {
faXmark,
faVideo,
} from "@fortawesome/pro-solid-svg-icons";
import { MediaUploadApi, EIntermediateFile } from "@storyteller/api";
import type { RefVideoAsset } from "../create-world-store";
import { UploaderStates } from "@storyteller/common";
import {
uploadVideo,
getVideoDuration,
} from "../../../components/prompt-box/upload-media";
import type { RefVideo } from "../../../components/prompt-box";
// A single optional reference video for splat generation. Uploads via the
// generic new-video endpoint and stores the returned media token.
// A single optional reference video for splat generation (mobile form band;
// the desktop prompt box renders the same ref in its reference deck).
const SLOT_CLASS =
"flex aspect-square w-14 items-center justify-center overflow-hidden rounded-lg border-2 border-dashed border-white/25 bg-white/5 transition-all hover:border-white/40 hover:bg-white/10";
@@ -19,8 +23,8 @@ export function ReferenceVideoSlot({
video,
onChange,
}: {
video?: RefVideoAsset;
onChange: (video?: RefVideoAsset) => void;
video?: RefVideo;
onChange: (video?: RefVideo) => void;
}) {
const inputRef = useRef<HTMLInputElement>(null);
const [uploading, setUploading] = useState(false);
@@ -30,41 +34,41 @@ export function ReferenceVideoSlot({
if (!file) return;
setUploading(true);
const previewUrl = URL.createObjectURL(file);
try {
const api = new MediaUploadApi();
const response = await api.UploadNewVideo({
blob: file,
fileName: file.name || `reference-video-${Date.now()}`,
uuid: crypto.randomUUID(),
maybe_title: "splat_reference_video",
is_intermediate_system_file: EIntermediateFile.false,
});
if (response?.success && response.data) {
onChange({
id: Math.random().toString(36).substring(7),
previewUrl,
mediaToken: response.data,
});
} else {
URL.revokeObjectURL(previewUrl);
}
} catch {
URL.revokeObjectURL(previewUrl);
} finally {
setUploading(false);
if (inputRef.current) inputRef.current.value = "";
}
const duration = await getVideoDuration(file);
await uploadVideo({
title: "splat_reference",
assetFile: file,
progressCallback: (state) => {
if (state.status === UploaderStates.success && state.data) {
onChange({
id: Math.random().toString(36).substring(7),
url: previewUrl,
file,
mediaToken: state.data,
duration,
});
setUploading(false);
} else if (
state.status === UploaderStates.assetError ||
state.status === UploaderStates.imageCreateError
) {
URL.revokeObjectURL(previewUrl);
setUploading(false);
}
},
});
if (inputRef.current) inputRef.current.value = "";
};
const handleRemove = () => {
if (video?.previewUrl) URL.revokeObjectURL(video.previewUrl);
if (video?.url.startsWith("blob:")) URL.revokeObjectURL(video.url);
onChange(undefined);
};
return (
// Title on the left, upload slot right-aligned + top-aligned (matches
// ImagePromptRow). Flat bottom on desktop so it seams into the prompt box.
<div className="glass flex items-start gap-3 rounded-2xl px-3 py-2 sm:rounded-t-2xl sm:rounded-b-none">
// ImagePromptRow).
<div className="glass flex items-start gap-3 rounded-2xl px-3 py-2">
<div className="flex grow flex-col gap-1 min-w-32">
<div className="flex items-center gap-2 text-white/90">
<FontAwesomeIcon icon={faVideo} className="h-3.5 w-3.5" />
@@ -82,7 +86,7 @@ export function ReferenceVideoSlot({
{video ? (
<div className="group relative aspect-square w-14 overflow-hidden rounded-lg border-2 border-white/30">
<video
src={video.previewUrl}
src={video.url}
muted
preload="metadata"
className="h-full w-full object-cover"
@@ -1,7 +1,7 @@
import { create } from "zustand";
import { persist } from "zustand/middleware";
import type { RecreatePayload } from "../../lib/recreate";
import type { RefImage } from "../../components/prompt-box";
import type { RefImage, RefVideo } from "../../components/prompt-box";
// A generated splat (3D world). `cdn_url` here is the .spz file.
export interface GeneratedAsset {
@@ -28,23 +28,17 @@ export type SplatUiState = {
disableRecaption: boolean | undefined;
};
// A stored reference video: media token + a local object-URL for preview. Not
// persisted (File handles / blob URLs aren't serializable).
export type RefVideoAsset = {
id: string;
previewUrl: string;
mediaToken: string;
};
type CreateWorldState = {
batches: SplatBatch[];
ui: SplatUiState;
referenceImages: RefImage[];
referenceVideo: RefVideoAsset | undefined;
// At most one guide video. Not persisted (File handles / blob URLs aren't
// serializable).
referenceVideos: RefVideo[];
pendingRecreate: RecreatePayload | null;
setUi: (patch: Partial<SplatUiState>) => void;
setReferenceImages: (images: RefImage[]) => void;
setReferenceVideo: (video: RefVideoAsset | undefined) => void;
setReferenceVideos: (videos: RefVideo[]) => void;
setPendingRecreate: (payload: RecreatePayload | null) => void;
consumePendingRecreate: () => RecreatePayload | null;
startBatch: (prompt: string, modelLabel: string) => string;
@@ -68,14 +62,14 @@ export const useCreateWorldStore = create<CreateWorldState>()(
batches: [],
ui: { ...DEFAULT_UI },
referenceImages: [],
referenceVideo: undefined,
referenceVideos: [],
pendingRecreate: null,
setUi: (patch) => set((s) => ({ ui: { ...s.ui, ...patch } })),
setReferenceImages: (images) => set({ referenceImages: images }),
setReferenceVideo: (video) => set({ referenceVideo: video }),
setReferenceVideos: (videos) => set({ referenceVideos: videos }),
setPendingRecreate: (payload) => set({ pendingRecreate: payload }),
@@ -113,8 +113,8 @@ export default function CreateWorld() {
const [isGenerating, setIsGenerating] = useState(false);
const referenceImages = useCreateWorldStore((s) => s.referenceImages);
const setReferenceImages = useCreateWorldStore((s) => s.setReferenceImages);
const referenceVideo = useCreateWorldStore((s) => s.referenceVideo);
const setReferenceVideo = useCreateWorldStore((s) => s.setReferenceVideo);
const referenceVideos = useCreateWorldStore((s) => s.referenceVideos);
const setReferenceVideos = useCreateWorldStore((s) => s.setReferenceVideos);
const [isImagePickerOpen, setIsImagePickerOpen] = useState(false);
const [pickerSelectedIds, setPickerSelectedIds] = useState<string[]>([]);
@@ -202,7 +202,7 @@ export default function CreateWorld() {
const estimatedCredits = useSplatCostEstimate({
model: selectedModel?.model ?? "",
referenceImageCount: referenceImages.length,
hasReferenceVideo: !!referenceVideo,
hasReferenceVideo: referenceVideos.length > 0,
isPanoramic: supportsPanorama ? isPanoramic : undefined,
disableRecaption: supportsDisableRecaption ? disableRecaption : undefined,
});
@@ -222,7 +222,9 @@ export default function CreateWorld() {
const canGenerate =
!!selectedModel &&
!isGenerating &&
(prompt.trim().length > 0 || referenceImages.length > 0 || !!referenceVideo);
(prompt.trim().length > 0 ||
referenceImages.length > 0 ||
referenceVideos.length > 0);
// ── Effects ──────────────────────────────────────────────────────────────
@@ -312,10 +314,12 @@ export default function CreateWorld() {
? referenceImageMediaTokens
: undefined,
referenceVideoMediaToken: supportsVideo
? referenceVideo?.mediaToken
? referenceVideos[0]?.mediaToken
: undefined,
isPanoramic: supportsPanorama ? isPanoramic : undefined,
disableRecaption: supportsDisableRecaption ? disableRecaption : undefined,
disableRecaption: supportsDisableRecaption
? disableRecaption
: undefined,
});
if (!result.success || !result.jobToken) {
@@ -368,7 +372,7 @@ export default function CreateWorld() {
supportsPanorama,
supportsDisableRecaption,
referenceImages,
referenceVideo,
referenceVideos,
isPanoramic,
disableRecaption,
startBatch,
@@ -394,7 +398,11 @@ export default function CreateWorld() {
</Tooltip>
)}
{supportsDisableRecaption && (
<Tooltip content="Use prompt exactly (skip recaption)" position="top" closeOnClick>
<Tooltip
content="Use prompt exactly (skip recaption)"
position="top"
closeOnClick
>
<ToggleButton
isActive={disableRecaption}
icon={faWandMagicSparkles}
@@ -407,8 +415,12 @@ export default function CreateWorld() {
</>
);
// Mobile-only band; on desktop the guide video lives in the reference deck.
const videoRow = supportsVideo ? (
<ReferenceVideoSlot video={referenceVideo} onChange={setReferenceVideo} />
<ReferenceVideoSlot
video={referenceVideos[0]}
onChange={(video) => setReferenceVideos(video ? [video] : [])}
/>
) : undefined;
// ── Mobile form ─────────────────────────────────────────────────────────
@@ -488,7 +500,7 @@ export default function CreateWorld() {
promptBox={
<div
ref={promptBoxRef}
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-5xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
className="animate-fade-in-up fixed bottom-2 sm:bottom-3 right-0 z-30 mx-auto max-w-6xl px-2 sm:px-4 transition-[left] duration-200 ease-linear"
style={{
animationDelay: "150ms",
left: "var(--ac-sidebar-offset, 0px)",
@@ -507,11 +519,18 @@ export default function CreateWorld() {
referenceImages={referenceImages}
onReferenceImagesChange={setReferenceImages}
onPickFromLibrary={() => setIsImagePickerOpen(true)}
onClearAllExtras={() => setReferenceVideo(undefined)}
hasClearableExtras={!!referenceVideo}
mediaReferenceRow={videoRow}
videoRefsSupported={supportsVideo}
referenceVideos={referenceVideos}
onReferenceVideosChange={setReferenceVideos}
maxVideoCount={1}
maxVideoRefDuration={Infinity}
modelSelector={
<Tooltip content="Model" position="top" className="z-50" closeOnClick>
<Tooltip
content="Model"
position="top"
className="z-50"
closeOnClick
>
<PopoverMenu
items={modelItems}
onSelect={handleModelChange}
@@ -521,7 +540,9 @@ export default function CreateWorld() {
richList
triggerIcon={
<img
src={getCreatorIconPathForModelId(selectedModel?.model ?? "")}
src={getCreatorIconPathForModelId(
selectedModel?.model ?? "",
)}
alt=""
className="h-4 w-4 icon-auto-contrast"
/>
@@ -8,6 +8,7 @@ import {
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import {
faArrowUpFromBracket,
faCube,
faImages,
faPlay,
faSpinnerThird,
@@ -87,7 +88,8 @@ export const DeckCard = ({
if (item.uploading) return;
if (item.kind === "audio") {
handleToggleAudio();
} else {
} else if (item.kind !== "mesh") {
// Mesh files have nothing to preview.
onClick?.();
}
};
@@ -99,6 +101,7 @@ export const DeckCard = ({
!item.uploading &&
"cursor-pointer hover:border-white/80 hover:cursor-zoom-in",
item.kind === "audio" && "hover:cursor-pointer",
item.kind === "mesh" && "hover:cursor-default",
animateIn && "animate-[deck-pop_180ms_ease-out]",
className,
)}
@@ -127,6 +130,14 @@ export const DeckCard = ({
)}
/>
)}
{item.kind === "mesh" && (
<div className="flex h-full w-full items-center justify-center">
<FontAwesomeIcon
icon={faCube}
className="h-5 w-5 text-base-fg/60 transition-colors group-hover:text-base-fg"
/>
</div>
)}
{item.kind === "audio" && (
<div className="flex h-full w-full items-center justify-center">
<FontAwesomeIcon
@@ -55,7 +55,7 @@ export const KeyframeCards = ({
>
<DeckStyles />
<KeyframeSlot
<DeckSlotCard
item={firstFrame}
label="First frame"
tiltClass={showLastFrame ? "-rotate-6" : ""}
@@ -83,7 +83,7 @@ export const KeyframeCards = ({
)}
</div>
<KeyframeSlot
<DeckSlotCard
item={lastFrame}
label="Last frame"
tiltClass="rotate-6"
@@ -102,17 +102,23 @@ export const KeyframeCards = ({
);
};
const KeyframeSlot = ({
/**
* One always-visible labeled reference slot: a dashed add card (with a
* per-slot add menu when there are multiple actions) that becomes a DeckCard
* once filled. Used for keyframes and for fixed named slots like the 3D
* multi-view angles; the caller owns the DeckPreviewModal fed by onPreview.
*/
export const DeckSlotCard = ({
item,
label,
tiltClass,
tiltClass = "",
addActions,
onRemove,
onPreview,
}: {
item?: DeckItem;
label: string;
tiltClass: string;
tiltClass?: string;
addActions: DeckAddAction[];
onRemove?: () => void;
onPreview: (item: DeckItem) => void;
@@ -129,7 +129,10 @@ export const ReferenceDeck = ({
);
const imageItems = useMemo(
() => items.filter((i) => i.kind === "image" && !i.uploading),
() =>
items.filter(
(i) => i.kind === "image" && !i.uploading && i.sortable !== false,
),
[items],
);
const allowReorder = !!onReorderImages && imageItems.length > 1;
@@ -308,7 +311,10 @@ export const ReferenceDeck = ({
strategy={rectSortingStrategy}
>
{items.map((item) =>
allowReorder && item.kind === "image" && !item.uploading ? (
allowReorder &&
item.kind === "image" &&
!item.uploading &&
item.sortable !== false ? (
<SortableDeckCard
key={item.id}
item={item}
@@ -378,14 +384,17 @@ export const ReferenceDeck = ({
>
<DeckStyles />
{/* Collapsed fan — fixed footprint so the textarea never reflows.
Only the card stack expands the panel on hover; the circular +
in front has its own hover menu. */}
{/* Collapsed fan — footprint tracks the card count (stack width plus
the circular +'s 12px overhang) so a single card doesn't reserve
the full 3-card fan width and push the textarea away. Only the
card stack expands the panel on hover; the circular + in front has
its own hover menu. */}
<div
className={twMerge(
"relative h-14 w-[92px] transition-opacity duration-150",
"relative h-14 transition-opacity duration-150",
expanded && "pointer-events-none opacity-0",
)}
style={{ width: `${(collapsedItems.length - 1) * 12 + 68}px` }}
>
<div
className="absolute inset-y-0 left-0"
@@ -1,6 +1,6 @@
import type { IconDefinition } from "@fortawesome/fontawesome-svg-core";
export type DeckItemKind = "image" | "video" | "audio";
export type DeckItemKind = "image" | "video" | "audio" | "mesh";
/**
* Neutral reference item consumed by the deck UI. Callers adapt their own
@@ -20,6 +20,9 @@ export interface DeckItem {
duration?: number;
/** True while its upload is in flight — blurred thumb + spinner overlay. */
uploading?: boolean;
/** False excludes an image card from drag-reorder (fixed slots like
* multi-view angles whose position is meaningful). Default true. */
sortable?: boolean;
}
/** One entry in a deck "+" add menu (upload, pick from library, ...). */
@@ -50,6 +50,12 @@ export interface UseDeckMediaOptions<
const randomId = () => Math.random().toString(36).substring(7);
// A non-finite duration cap means "no limit" — leave it out of the message.
const videoLimitMessage = (maxVideos: number, maxTotalSec: number) =>
isFinite(maxTotalSec)
? `Max ${maxVideos} videos / ${maxTotalSec}s total`
: `Max ${maxVideos} video${maxVideos === 1 ? "" : "s"}`;
const randomTitle = (prefix: string) =>
`${prefix}-${Math.random().toString(36).substring(2, 15)}`;
@@ -291,7 +297,7 @@ export function useDeckMedia<
const baseVideos = [...referenceVideos];
const availableSlots = Math.max(0, maxVideos - baseVideos.length);
if (availableSlots <= 0) {
toast.error(`Max ${maxVideos} videos / ${maxVideoTotalSec}s total`, {
toast.error(videoLimitMessage(maxVideos, maxVideoTotalSec), {
id: "video-ref-limit",
});
return;
@@ -443,8 +449,9 @@ export function useDeckMedia<
if (audios.length > 0 && setReferenceAudios) processAudioFiles(audios);
};
// maxImages of 0 means the page's model takes no image refs at all.
const anyUploadAccept = [
"image/*",
...(maxImages > 0 ? ["image/*"] : []),
...(setReferenceVideos ? ["video/mp4", ".mp4"] : []),
...(setReferenceAudios ? ["audio/*"] : []),
].join(",");
@@ -479,7 +486,7 @@ export function useDeckMedia<
const baseVideos = [...referenceVideos];
const availableSlots = Math.max(0, maxVideos - baseVideos.length);
if (availableSlots <= 0) {
toast.error(`Max ${maxVideos} videos / ${maxVideoTotalSec}s total`, {
toast.error(videoLimitMessage(maxVideos, maxVideoTotalSec), {
id: "video-ref-limit",
});
handleGalleryClose();