mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-05 17:58:55 +00:00
more controls for seedance 2.0 (#1052)
Co-authored-by: Brandon Thomas <echelon@gmail.com>
This commit is contained in:
BFlatandBrandon Thomas authored and 
GitHub committed
1 parent
f36a5d5e02
commit
3e0bf8ebfd
15 files changed
+833
-263
No files matched your search
Generated
+1
@@ -738,6 +738,7 @@ dependencies = [
|
||||
name = "artcraft_api_defs"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"enums",
|
||||
"enums_public",
|
||||
|
||||
@@ -36,3 +36,4 @@ serde_json.workspace = true
|
||||
utoipa = { version = "4", features = ["chrono", "actix_extras"] }
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
+17
@@ -66,3 +66,20 @@ pub struct Seedance2p0MultiFunctionVideoGenResponse {
|
||||
/// All inference job tokens for the batch (includes `inference_job_token` as the first entry).
|
||||
pub all_inference_job_tokens: Vec<InferenceJobToken>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn aspect_to_string(input: Seedance2p0AspectRatio) -> anyhow::Result<String> {
|
||||
let val = serde_json::to_string(&input)?;
|
||||
Ok(val.replace("\"", ""))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn just_testing_snake_case() -> anyhow::Result<()> {
|
||||
// Ugh, not sure what these are...
|
||||
assert_eq!(aspect_to_string(Seedance2p0AspectRatio::Landscape16x9)?, "landscape16x9");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ export const defaultModelForPage = (models: Model[], page: ModelPage): Model =>
|
||||
imageModel = IMAGE_MODELS_BY_ID.get("nano_banana_pro");
|
||||
break;
|
||||
case ModelPage.ImageToVideo:
|
||||
imageModel = VIDEO_MODELS_BY_ID.get("seedance_1_0_lite");
|
||||
imageModel = VIDEO_MODELS_BY_ID.get("seedance_2p0");
|
||||
break;
|
||||
case ModelPage.Canvas2D:
|
||||
imageModel = IMAGE_MODELS_BY_ID.get("gpt_image_1p5");
|
||||
|
||||
@@ -60,6 +60,7 @@ interface ImagePromptRowProps {
|
||||
uploadImage?: UploadImageFn;
|
||||
onImageClick?: (image: RefImage) => void;
|
||||
isVideo?: boolean;
|
||||
isReferenceMode?: boolean;
|
||||
endFrameImage?: RefImage;
|
||||
setEndFrameImage?: (image?: RefImage) => void;
|
||||
allowUploadEnd?: boolean;
|
||||
@@ -77,6 +78,7 @@ export const ImagePromptRow = ({
|
||||
uploadImage,
|
||||
onImageClick,
|
||||
isVideo,
|
||||
isReferenceMode,
|
||||
endFrameImage,
|
||||
setEndFrameImage,
|
||||
allowUploadEnd,
|
||||
@@ -116,7 +118,7 @@ export const ImagePromptRow = ({
|
||||
})
|
||||
);
|
||||
|
||||
const SortableImage = ({ image }: { image: RefImage }) => {
|
||||
const SortableImage = ({ image, index }: { image: RefImage; index: number }) => {
|
||||
const {
|
||||
attributes,
|
||||
listeners,
|
||||
@@ -151,6 +153,11 @@ export const ImagePromptRow = ({
|
||||
alt="Reference"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
{isReferenceMode && (
|
||||
<div className="absolute bottom-0 left-0 right-0 flex items-center justify-center bg-black/60 py-0.5 text-[11px] font-bold text-white">
|
||||
{index + 1}
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
@@ -413,8 +420,12 @@ export const ImagePromptRow = ({
|
||||
<div className="flex items-center gap-2 opacity-90 text-base-fg">
|
||||
<FontAwesomeIcon icon={faImage} className="h-3.5 w-3.5" />
|
||||
<span className="text-sm font-medium flex items-center gap-1.5">
|
||||
{isVideo ? "Starting Frame" : "Image Prompts"}
|
||||
{!isVideo && (
|
||||
{isVideo
|
||||
? isReferenceMode
|
||||
? "Reference Images"
|
||||
: "Starting Frame"
|
||||
: "Image Prompts"}
|
||||
{(!isVideo || isReferenceMode) && (
|
||||
<span className="text-base-fg/60 font-semibold">
|
||||
({usedSlotsRender}/{maxImagePromptCount})
|
||||
</span>
|
||||
@@ -422,7 +433,11 @@ export const ImagePromptRow = ({
|
||||
</span>
|
||||
</div>
|
||||
<span className="text-[13px] text-base-fg/60">
|
||||
{isVideo ? "Animate an image" : "Use the elements of an image"}
|
||||
{isVideo
|
||||
? isReferenceMode
|
||||
? "Upload images"
|
||||
: "Animate an image"
|
||||
: "Use the elements of an image"}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -441,15 +456,15 @@ export const ImagePromptRow = ({
|
||||
>
|
||||
{referenceImages
|
||||
.slice(0, Math.max(0, maxImagePromptCount))
|
||||
.map((image) => (
|
||||
<SortableImage key={image.id} image={image} />
|
||||
.map((image, index) => (
|
||||
<SortableImage key={image.id} image={image} index={index} />
|
||||
))}
|
||||
</SortableContext>
|
||||
</DndContext>
|
||||
) : (
|
||||
referenceImages
|
||||
.slice(0, Math.max(0, maxImagePromptCount))
|
||||
.map((image) => (
|
||||
.map((image, index) => (
|
||||
<div
|
||||
key={image.id}
|
||||
className="glass relative aspect-square overflow-hidden rounded-lg w-14 border-2 border-white/30 hover:border-white/80 transition-all group cursor-pointer hover:cursor-zoom-in"
|
||||
@@ -460,6 +475,11 @@ export const ImagePromptRow = ({
|
||||
alt="Reference"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
{isReferenceMode && (
|
||||
<div className="absolute bottom-0 left-0 right-0 flex items-center justify-center bg-black/60 py-0.5 text-[11px] font-bold text-white">
|
||||
{index + 1}
|
||||
</div>
|
||||
)}
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
|
||||
@@ -19,13 +19,7 @@ import {
|
||||
faSpinnerThird,
|
||||
faExpand,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
faRectangle,
|
||||
faSquare,
|
||||
faRectangleVertical,
|
||||
} from "@fortawesome/pro-regular-svg-icons";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { IconProp } from "@fortawesome/fontawesome-svg-core";
|
||||
import { CommonAspectRatio, ImageModel } from "@storyteller/model-list";
|
||||
import { usePromptImageStore, RefImage } from "./promptStore";
|
||||
import { gtagEvent } from "@storyteller/google-analytics";
|
||||
@@ -33,6 +27,7 @@ import { twMerge } from "tailwind-merge";
|
||||
import { ImagePromptRow } from "./ImagePromptRow";
|
||||
import { GenerationProvider } from "@storyteller/api-enums";
|
||||
import { AspectRatioPicker } from "./common/AspectRatioPicker";
|
||||
import { AspectRatioIcon } from "./common/AspectRatioIcon";
|
||||
import { GenerationCountPicker } from "./common/GenerationCountPicker";
|
||||
import { ResolutionPicker } from "./common/ResolutionPicker";
|
||||
import { CommonResolution } from "@storyteller/model-list";
|
||||
@@ -134,17 +129,17 @@ export const PromptBoxImage = ({
|
||||
{
|
||||
label: "Wide",
|
||||
selected: aspectRatio === "wide",
|
||||
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
|
||||
icon: <AspectRatioIcon ratio={[16, 10]} />,
|
||||
},
|
||||
{
|
||||
label: "Tall",
|
||||
selected: aspectRatio === "tall",
|
||||
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
|
||||
icon: <AspectRatioIcon ratio={[10, 16]} />,
|
||||
},
|
||||
{
|
||||
label: "Square",
|
||||
selected: aspectRatio === "square",
|
||||
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
|
||||
icon: <AspectRatioIcon ratio={[1, 1]} />,
|
||||
},
|
||||
]);
|
||||
const [resolutionList, setResolutionList] = useState<PopoverItem[]>([
|
||||
@@ -319,11 +314,18 @@ export const PromptBoxImage = ({
|
||||
}
|
||||
};
|
||||
|
||||
const getCurrentResolutionIcon = () => {
|
||||
const getCurrentLegacyAspectRatioIcon = () => {
|
||||
const selected = aspectRatioList.find((item) => item.selected);
|
||||
if (!selected || !selected.icon) return faRectangle;
|
||||
const iconElement = selected.icon as React.ReactElement<{ icon: IconProp }>;
|
||||
return iconElement.props.icon;
|
||||
switch (selected?.label?.toLowerCase()) {
|
||||
case "wide":
|
||||
return <AspectRatioIcon ratio={[16, 10]} />;
|
||||
case "tall":
|
||||
return <AspectRatioIcon ratio={[10, 16]} />;
|
||||
case "square":
|
||||
return <AspectRatioIcon ratio={[1, 1]} />;
|
||||
default:
|
||||
return <AspectRatioIcon ratio={[16, 10]} />;
|
||||
}
|
||||
};
|
||||
|
||||
const getCurrentLegacyAspectRatio = (): EnqueueTextToImageSize => {
|
||||
@@ -481,12 +483,7 @@ export const PromptBoxImage = ({
|
||||
mode="toggle"
|
||||
panelTitle="Aspect Ratio (Legacy)"
|
||||
showIconsInList
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon
|
||||
icon={getCurrentResolutionIcon()}
|
||||
className="h-4 w-4"
|
||||
/>
|
||||
}
|
||||
triggerIcon={getCurrentLegacyAspectRatioIcon()}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
@@ -16,25 +16,21 @@ import {
|
||||
faSparkles,
|
||||
faSpinnerThird,
|
||||
faWaveformLines,
|
||||
faClock,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {faCircleInfo} from "@fortawesome/pro-regular-svg-icons";
|
||||
import {
|
||||
faRectangle,
|
||||
faSquare,
|
||||
faRectangleVertical,
|
||||
} from "@fortawesome/pro-regular-svg-icons";
|
||||
import { faCircleInfo } from "@fortawesome/pro-regular-svg-icons";
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
import { IconProp } from "@fortawesome/fontawesome-svg-core";
|
||||
import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal";
|
||||
import {
|
||||
SizeIconOption,
|
||||
SizeOption,
|
||||
VideoModel,
|
||||
} from "@storyteller/model-list";
|
||||
import { usePromptVideoStore, RefImage } from "./promptStore";
|
||||
import { usePromptVideoStore, RefImage, VideoInputMode } from "./promptStore";
|
||||
import { gtagEvent } from "@storyteller/google-analytics";
|
||||
import { ImagePromptRow } from "./ImagePromptRow";
|
||||
import type { UploadImageFn } from "./ImagePromptRow";
|
||||
import { AspectRatioIcon } from "./common/AspectRatioIcon";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
import { GenerationProvider } from "@storyteller/api-enums";
|
||||
@@ -105,6 +101,12 @@ export const PromptBoxVideo = ({
|
||||
);
|
||||
const resolution = usePromptVideoStore((s) => s.resolution);
|
||||
const setResolution = usePromptVideoStore((s) => s.setResolution);
|
||||
const aspectRatio = usePromptVideoStore((s) => s.aspectRatio);
|
||||
const setAspectRatio = usePromptVideoStore((s) => s.setAspectRatio);
|
||||
const duration = usePromptVideoStore((s) => s.duration);
|
||||
const setDuration = usePromptVideoStore((s) => s.setDuration);
|
||||
const inputMode = usePromptVideoStore((s) => s.inputMode);
|
||||
const setInputMode = usePromptVideoStore((s) => s.setInputMode);
|
||||
const [isEnqueueing, setIsEnqueueing] = useState(false);
|
||||
const [isFocused, setIsFocused] = useState(false);
|
||||
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
|
||||
@@ -124,68 +126,30 @@ export const PromptBoxVideo = ({
|
||||
uploadingImages.length > 0;
|
||||
|
||||
// TODO: Get rid of default resolutions. Just disable it if not present.
|
||||
let resolutionOptions: PopoverItem[];
|
||||
let aspectRatioOptions: PopoverItem[];
|
||||
|
||||
const buildAspectRatioOptions = (options: SizeOption[]): PopoverItem[] => {
|
||||
const currentExists = options.some(
|
||||
(option) => option.textLabel === aspectRatio,
|
||||
);
|
||||
const useFirstOption = !currentExists;
|
||||
|
||||
return options.map((option, index) => ({
|
||||
label: option.textLabel,
|
||||
selected:
|
||||
option.textLabel === aspectRatio || (useFirstOption && index === 0),
|
||||
icon: <AspectRatioIcon sizeIcon={option.icon} />,
|
||||
}));
|
||||
};
|
||||
|
||||
if (!!selectedModel?.sizeOptions && selectedModel.sizeOptions.length > 0) {
|
||||
// When switching to a new model, the existing resolution might not be correct.
|
||||
// This is a gross and nasty hack to handle the case where the resolution is not found.
|
||||
const resolutionExists = selectedModel.sizeOptions.some(
|
||||
(option) => option.textLabel === resolution,
|
||||
);
|
||||
const useFirstOption = !resolutionExists;
|
||||
|
||||
resolutionOptions = selectedModel.sizeOptions.map((option, index) => {
|
||||
let faIcon = faRectangle;
|
||||
switch (option.icon) {
|
||||
case SizeIconOption.Landscape:
|
||||
faIcon = faRectangle;
|
||||
break;
|
||||
case SizeIconOption.Portrait:
|
||||
faIcon = faRectangleVertical;
|
||||
break;
|
||||
case SizeIconOption.Square:
|
||||
faIcon = faSquare;
|
||||
break;
|
||||
}
|
||||
const icon = <FontAwesomeIcon icon={faIcon} className="h-4 w-4" />;
|
||||
return {
|
||||
label: option.textLabel,
|
||||
selected:
|
||||
option.textLabel === resolution || (useFirstOption && index === 0),
|
||||
icon: icon,
|
||||
};
|
||||
});
|
||||
aspectRatioOptions = buildAspectRatioOptions(selectedModel.sizeOptions);
|
||||
} else {
|
||||
const resolutionExists = DEFAULT_RESOLUTIONS.some(
|
||||
(option) => option.textLabel === resolution,
|
||||
);
|
||||
const useFirstOption = !resolutionExists;
|
||||
|
||||
resolutionOptions = DEFAULT_RESOLUTIONS.map((option, index) => {
|
||||
let faIcon = faRectangle;
|
||||
switch (option.icon) {
|
||||
case SizeIconOption.Landscape:
|
||||
faIcon = faRectangle;
|
||||
break;
|
||||
case SizeIconOption.Portrait:
|
||||
faIcon = faRectangleVertical;
|
||||
break;
|
||||
case SizeIconOption.Square:
|
||||
faIcon = faSquare;
|
||||
break;
|
||||
}
|
||||
const icon = <FontAwesomeIcon icon={faIcon} className="h-4 w-4" />;
|
||||
return {
|
||||
label: option.textLabel,
|
||||
selected:
|
||||
option.textLabel === resolution || (useFirstOption && index === 0),
|
||||
icon: icon,
|
||||
};
|
||||
});
|
||||
aspectRatioOptions = buildAspectRatioOptions(DEFAULT_RESOLUTIONS);
|
||||
}
|
||||
|
||||
const [resolutionList, setResolutionList] =
|
||||
useState<PopoverItem[]>(resolutionOptions);
|
||||
const [aspectRatioList, setAspectRatioList] =
|
||||
useState<PopoverItem[]>(aspectRatioOptions);
|
||||
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
@@ -212,17 +176,174 @@ export const PromptBoxVideo = ({
|
||||
onImageRowVisibilityChange?.(isImageRowVisible);
|
||||
}, [isImageRowVisible, onImageRowVisibilityChange]);
|
||||
|
||||
const handleResolutionSelect = (selectedItem: PopoverItem) => {
|
||||
console.log(">>>> handleResolutionSelect", selectedItem);
|
||||
setResolution(selectedItem.label as any);
|
||||
setResolutionList((prev) =>
|
||||
resolutionOptions.map((item) => ({
|
||||
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
|
||||
setAspectRatio(selectedItem.label);
|
||||
setAspectRatioList((prev) =>
|
||||
aspectRatioOptions.map((item) => ({
|
||||
...item,
|
||||
selected: item.label === selectedItem.label,
|
||||
})),
|
||||
);
|
||||
};
|
||||
|
||||
// Sync duration with model default when switching models
|
||||
useEffect(() => {
|
||||
if (selectedModel?.durationOptions && selectedModel.defaultDuration) {
|
||||
if (
|
||||
duration === null ||
|
||||
!selectedModel.durationOptions.includes(duration)
|
||||
) {
|
||||
setDuration(selectedModel.defaultDuration);
|
||||
}
|
||||
} else if (duration !== null) {
|
||||
setDuration(null);
|
||||
}
|
||||
}, [selectedModel]);
|
||||
|
||||
// Sync resolution with model default when switching models
|
||||
useEffect(() => {
|
||||
if (selectedModel?.resolutionOptions && selectedModel.defaultResolution) {
|
||||
if (!selectedModel.resolutionOptions.includes(resolution as string)) {
|
||||
setResolution(selectedModel.defaultResolution);
|
||||
}
|
||||
}
|
||||
}, [selectedModel]);
|
||||
|
||||
// Reset input mode when switching to a model that doesn't support reference
|
||||
useEffect(() => {
|
||||
if (!selectedModel?.supportsReferenceMode && inputMode === "reference") {
|
||||
setInputMode("keyframe");
|
||||
}
|
||||
}, [selectedModel]);
|
||||
|
||||
const durationOptions: PopoverItem[] | null = selectedModel?.durationOptions
|
||||
? selectedModel.durationOptions.map((d) => ({
|
||||
label: `${d}s`,
|
||||
selected: d === (duration ?? selectedModel.defaultDuration),
|
||||
}))
|
||||
: null;
|
||||
|
||||
const resolutionPickerOptions: PopoverItem[] | null =
|
||||
selectedModel?.resolutionOptions
|
||||
? selectedModel.resolutionOptions.map((r) => ({
|
||||
label: r,
|
||||
selected: r === resolution,
|
||||
}))
|
||||
: null;
|
||||
|
||||
const handleResolutionSelect = (selectedItem: PopoverItem) => {
|
||||
setResolution(selectedItem.label);
|
||||
};
|
||||
|
||||
const handleDurationSelect = (selectedItem: PopoverItem) => {
|
||||
const seconds = parseInt(selectedItem.label);
|
||||
setDuration(seconds);
|
||||
};
|
||||
|
||||
const inputModeOptions: PopoverItem[] | null =
|
||||
selectedModel?.supportsReferenceMode
|
||||
? [
|
||||
{
|
||||
label: "Keyframe",
|
||||
description: "First/Last frame",
|
||||
selected: inputMode === "keyframe",
|
||||
},
|
||||
{
|
||||
label: "Reference",
|
||||
description: "Multi-media ref",
|
||||
selected: inputMode === "reference",
|
||||
},
|
||||
]
|
||||
: null;
|
||||
|
||||
const handleInputModeSelect = (selectedItem: PopoverItem) => {
|
||||
const mode: VideoInputMode =
|
||||
selectedItem.label === "Reference" ? "reference" : "keyframe";
|
||||
setInputMode(mode);
|
||||
// Clear images when switching modes to avoid stale state
|
||||
if (mode === "reference") {
|
||||
setEndFrameImage(undefined);
|
||||
}
|
||||
};
|
||||
|
||||
const isReferenceMode =
|
||||
inputMode === "reference" && !!selectedModel?.supportsReferenceMode;
|
||||
const maxImageCount = isReferenceMode
|
||||
? (selectedModel?.maxReferenceImages ?? 3)
|
||||
: 1;
|
||||
|
||||
const highlightRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Sync scroll between textarea and highlight overlay
|
||||
const handleScroll = () => {
|
||||
if (highlightRef.current && textareaRef.current) {
|
||||
highlightRef.current.scrollTop = textareaRef.current.scrollTop;
|
||||
}
|
||||
};
|
||||
|
||||
// Color palette for @Image mentions
|
||||
const MENTION_COLORS = [
|
||||
"rgb(96, 165, 250)", // blue
|
||||
"rgb(251, 146, 60)", // orange
|
||||
"rgb(167, 139, 250)", // purple
|
||||
"rgb(52, 211, 153)", // green
|
||||
"rgb(251, 113, 133)", // pink
|
||||
];
|
||||
|
||||
const renderHighlightedPrompt = () => {
|
||||
if (!isReferenceMode || referenceImages.length === 0) return null;
|
||||
const parts = prompt.split(/(@Image\d+)/g);
|
||||
return parts.map((part, i) => {
|
||||
const match = part.match(/^@Image(\d+)$/);
|
||||
if (match) {
|
||||
const imgIndex = parseInt(match[1]) - 1;
|
||||
const color = MENTION_COLORS[imgIndex % MENTION_COLORS.length];
|
||||
return (
|
||||
<span key={i} style={{ color, fontWeight: 600 }}>
|
||||
{part}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
return <span key={i}>{part}</span>;
|
||||
});
|
||||
};
|
||||
|
||||
// @-mention autocomplete state
|
||||
const [mentionOpen, setMentionOpen] = useState(false);
|
||||
const [mentionFilter, setMentionFilter] = useState("");
|
||||
const [mentionIndex, setMentionIndex] = useState(0);
|
||||
const mentionAnchorRef = useRef<number | null>(null);
|
||||
|
||||
const mentionItems = isReferenceMode
|
||||
? referenceImages
|
||||
.map((img, i) => ({
|
||||
label: `@Image${i + 1}`,
|
||||
image: img,
|
||||
}))
|
||||
.filter((item) =>
|
||||
mentionFilter
|
||||
? item.label.toLowerCase().includes(mentionFilter.toLowerCase())
|
||||
: true,
|
||||
)
|
||||
: [];
|
||||
|
||||
const insertMention = (label: string) => {
|
||||
const textarea = textareaRef.current;
|
||||
if (!textarea || mentionAnchorRef.current === null) return;
|
||||
const before = prompt.slice(0, mentionAnchorRef.current);
|
||||
const after = prompt.slice(textarea.selectionStart);
|
||||
const next = before + label + " " + after;
|
||||
setPrompt(next);
|
||||
setMentionOpen(false);
|
||||
setMentionFilter("");
|
||||
mentionAnchorRef.current = null;
|
||||
requestAnimationFrame(() => {
|
||||
const pos = before.length + label.length + 1;
|
||||
textarea.setSelectionRange(pos, pos);
|
||||
textarea.focus();
|
||||
});
|
||||
};
|
||||
|
||||
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
|
||||
e.preventDefault();
|
||||
const pastedText = e.clipboardData.getData("text");
|
||||
@@ -238,7 +359,31 @@ export const PromptBoxVideo = ({
|
||||
};
|
||||
|
||||
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
||||
setPrompt(e.target.value);
|
||||
const value = e.target.value;
|
||||
const cursorPos = e.target.selectionStart;
|
||||
setPrompt(value);
|
||||
|
||||
if (isReferenceMode && referenceImages.length > 0) {
|
||||
// Find the last '@' before cursor that could be a mention trigger
|
||||
const textBeforeCursor = value.slice(0, cursorPos);
|
||||
const lastAtIndex = textBeforeCursor.lastIndexOf("@");
|
||||
|
||||
if (lastAtIndex !== -1) {
|
||||
const textAfterAt = textBeforeCursor.slice(lastAtIndex + 1);
|
||||
// Only trigger if no space after @ (still typing the mention)
|
||||
if (!textAfterAt.includes(" ")) {
|
||||
mentionAnchorRef.current = lastAtIndex;
|
||||
setMentionFilter("@" + textAfterAt);
|
||||
setMentionOpen(true);
|
||||
setMentionIndex(0);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setMentionOpen(false);
|
||||
setMentionFilter("");
|
||||
mentionAnchorRef.current = null;
|
||||
};
|
||||
|
||||
const handleEnqueue = async () => {
|
||||
@@ -268,9 +413,12 @@ export const PromptBoxVideo = ({
|
||||
? crypto.randomUUID()
|
||||
: Math.random().toString(36).slice(2);
|
||||
|
||||
const isRefMode =
|
||||
inputMode === "reference" && !!selectedModel.supportsReferenceMode;
|
||||
|
||||
let imageMediaToken = undefined;
|
||||
|
||||
if (referenceImages.length > 0) {
|
||||
if (!isRefMode && referenceImages.length > 0) {
|
||||
imageMediaToken = referenceImages[0].mediaToken;
|
||||
}
|
||||
|
||||
@@ -285,7 +433,9 @@ export const PromptBoxVideo = ({
|
||||
model: selectedModel,
|
||||
image_media_token: imageMediaToken,
|
||||
prompt: prompt,
|
||||
end_frame_image_media_token: endFrameImage?.mediaToken,
|
||||
end_frame_image_media_token: isRefMode
|
||||
? undefined
|
||||
: endFrameImage?.mediaToken,
|
||||
frontend_caller: "image_to_video",
|
||||
frontend_subscriber_id: subscriberId,
|
||||
};
|
||||
@@ -298,6 +448,18 @@ export const PromptBoxVideo = ({
|
||||
request.generate_audio = !!generateWithSound;
|
||||
}
|
||||
|
||||
// Pass reference image tokens in reference mode
|
||||
if (isRefMode && referenceImages.length > 0) {
|
||||
request.reference_image_media_tokens = referenceImages.map(
|
||||
(img) => img.mediaToken,
|
||||
);
|
||||
}
|
||||
|
||||
// Pass duration if model supports it
|
||||
if (selectedModel.durationOptions && duration !== null) {
|
||||
request.duration_seconds = duration;
|
||||
}
|
||||
|
||||
switch (selectedModel?.tauriId) {
|
||||
case "grok_video":
|
||||
request.grok_aspect_ratio = getGrokAspectRatio();
|
||||
@@ -307,6 +469,17 @@ export const PromptBoxVideo = ({
|
||||
request.sora_orientation =
|
||||
resolution === "720p" ? "landscape" : "portrait";
|
||||
break;
|
||||
|
||||
case "seedance_2p0": {
|
||||
const selectedOption = selectedModel.sizeOptions?.find(
|
||||
(option) => option.textLabel === aspectRatio,
|
||||
);
|
||||
if (selectedOption) {
|
||||
request.aspect_ratio =
|
||||
selectedOption.tauriValue as typeof request.aspect_ratio;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await EnqueueImageToVideo(request);
|
||||
@@ -316,14 +489,40 @@ export const PromptBoxVideo = ({
|
||||
setIsEnqueueing(false);
|
||||
};
|
||||
|
||||
const getCurrentResolutionIcon = () => {
|
||||
const selected = resolutionList.find((item) => item.selected);
|
||||
if (!selected || !selected.icon) return faRectangle;
|
||||
const iconElement = selected.icon as React.ReactElement<{ icon: IconProp }>;
|
||||
return iconElement.props.icon;
|
||||
const getCurrentAspectRatioIcon = (): SizeIconOption => {
|
||||
const selectedLabel = aspectRatioList.find((item) => item.selected)?.label;
|
||||
const allOptions = selectedModel?.sizeOptions ?? DEFAULT_RESOLUTIONS;
|
||||
const match = allOptions.find((o) => o.textLabel === selectedLabel);
|
||||
return match?.icon ?? SizeIconOption.Landscape;
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
// Handle mention dropdown navigation
|
||||
if (mentionOpen && mentionItems.length > 0) {
|
||||
if (e.key === "ArrowDown") {
|
||||
e.preventDefault();
|
||||
setMentionIndex((prev) => (prev + 1) % mentionItems.length);
|
||||
return;
|
||||
}
|
||||
if (e.key === "ArrowUp") {
|
||||
e.preventDefault();
|
||||
setMentionIndex((prev) =>
|
||||
prev <= 0 ? mentionItems.length - 1 : prev - 1,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (e.key === "Enter" || e.key === "Tab") {
|
||||
e.preventDefault();
|
||||
insertMention(mentionItems[mentionIndex].label);
|
||||
return;
|
||||
}
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
setMentionOpen(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
|
||||
@@ -343,7 +542,7 @@ export const PromptBoxVideo = ({
|
||||
// NB: This function was just written to give us better type safety.
|
||||
// There has to be a cleaner appraoach.
|
||||
const maybeAspectRatio = selectedModel?.sizeOptions?.find(
|
||||
(option) => option.textLabel === resolution,
|
||||
(option) => option.textLabel === aspectRatio,
|
||||
)?.tauriValue;
|
||||
|
||||
switch (maybeAspectRatio) {
|
||||
@@ -386,7 +585,8 @@ export const PromptBoxVideo = ({
|
||||
<ImagePromptRow
|
||||
visible={true}
|
||||
isVideo={true}
|
||||
maxImagePromptCount={1}
|
||||
isReferenceMode={isReferenceMode}
|
||||
maxImagePromptCount={maxImageCount}
|
||||
allowUpload={true}
|
||||
referenceImages={referenceImages}
|
||||
setReferenceImages={setReferenceImages}
|
||||
@@ -401,10 +601,10 @@ export const PromptBoxVideo = ({
|
||||
setIsModalOpen(true);
|
||||
}}
|
||||
uploadImage={uploadImage}
|
||||
endFrameImage={endFrameImage}
|
||||
setEndFrameImage={setEndFrameImage}
|
||||
allowUploadEnd={!!selectedModel?.endFrame}
|
||||
showEndFrameSection={!!selectedModel?.endFrame}
|
||||
endFrameImage={isReferenceMode ? undefined : endFrameImage}
|
||||
setEndFrameImage={isReferenceMode ? undefined : setEndFrameImage}
|
||||
allowUploadEnd={!isReferenceMode && !!selectedModel?.endFrame}
|
||||
showEndFrameSection={!isReferenceMode && !!selectedModel?.endFrame}
|
||||
/>
|
||||
)}
|
||||
<div
|
||||
@@ -416,7 +616,38 @@ export const PromptBoxVideo = ({
|
||||
: "ring-1 ring-transparent",
|
||||
)}
|
||||
>
|
||||
<div className="flex justify-center gap-2">
|
||||
<div className="relative flex justify-center gap-2">
|
||||
{/* @-mention autocomplete dropdown */}
|
||||
{mentionOpen && mentionItems.length > 0 && (
|
||||
<div className="absolute bottom-full left-0 z-50 mb-1 w-64 overflow-hidden rounded-lg border border-white/10 bg-ui-controls shadow-lg backdrop-blur-xl">
|
||||
<div className="px-3 py-1.5 text-[11px] font-semibold uppercase tracking-wider text-base-fg/50">
|
||||
Reference Files
|
||||
</div>
|
||||
{mentionItems.map((item, i) => (
|
||||
<button
|
||||
key={item.label}
|
||||
className={twMerge(
|
||||
"flex w-full items-center gap-2.5 px-3 py-2 text-sm text-base-fg transition-colors cursor-pointer",
|
||||
i === mentionIndex ? "bg-white/10" : "hover:bg-white/5",
|
||||
)}
|
||||
onMouseDown={(e) => {
|
||||
e.preventDefault();
|
||||
insertMention(item.label);
|
||||
}}
|
||||
onMouseEnter={() => setMentionIndex(i)}
|
||||
>
|
||||
<div className="h-8 w-8 flex-shrink-0 overflow-hidden rounded-md border border-white/20">
|
||||
<img
|
||||
src={item.image.url}
|
||||
alt={item.label}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
</div>
|
||||
<span className="font-medium">{item.label}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
{/* Hide the Add image button for video for now */}
|
||||
{/* <Tooltip
|
||||
content="Add Image"
|
||||
@@ -448,59 +679,114 @@ export const PromptBoxVideo = ({
|
||||
</Button>
|
||||
</Tooltip> */}
|
||||
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
rows={1}
|
||||
placeholder="Describe what you want to happen in the video..."
|
||||
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-base-fg placeholder-base-fg/60 focus:outline-none"
|
||||
value={prompt}
|
||||
onChange={handleChange}
|
||||
onPaste={handlePaste}
|
||||
onKeyDown={handleKeyDown}
|
||||
onFocus={() => setIsFocused(true)}
|
||||
onBlur={() => setIsFocused(false)}
|
||||
/>
|
||||
<div className="relative flex-1">
|
||||
{isReferenceMode && referenceImages.length > 0 && (
|
||||
<div
|
||||
ref={highlightRef}
|
||||
aria-hidden
|
||||
className="text-md pointer-events-none absolute inset-0 max-h-[5.5em] overflow-y-auto whitespace-pre-wrap break-words rounded pb-2 pr-2 pt-1 text-base-fg"
|
||||
>
|
||||
{renderHighlightedPrompt()}
|
||||
</div>
|
||||
)}
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
rows={1}
|
||||
placeholder={
|
||||
isReferenceMode
|
||||
? "Use @Image1, @Image2... to reference your uploaded images in the prompt..."
|
||||
: "Describe what you want to happen in the video..."
|
||||
}
|
||||
className={twMerge(
|
||||
"text-md relative mb-2 max-h-[5.5em] w-full resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 placeholder-base-fg/60 focus:outline-none",
|
||||
isReferenceMode && referenceImages.length > 0
|
||||
? "text-transparent caret-base-fg"
|
||||
: "text-base-fg",
|
||||
)}
|
||||
value={prompt}
|
||||
onChange={handleChange}
|
||||
onPaste={handlePaste}
|
||||
onKeyDown={handleKeyDown}
|
||||
onScroll={handleScroll}
|
||||
onFocus={() => setIsFocused(true)}
|
||||
onBlur={() => setIsFocused(false)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-2 flex items-center justify-between gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Tooltip
|
||||
content="Resolution"
|
||||
content="Aspect Ratio"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick={true}
|
||||
>
|
||||
<PopoverMenu
|
||||
items={resolutionOptions}
|
||||
onSelect={handleResolutionSelect}
|
||||
items={aspectRatioOptions}
|
||||
onSelect={handleAspectRatioSelect}
|
||||
mode="toggle"
|
||||
panelTitle="Resolution"
|
||||
panelTitle="Aspect Ratio"
|
||||
showIconsInList
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon
|
||||
icon={getCurrentResolutionIcon()}
|
||||
className="h-4 w-4"
|
||||
/>
|
||||
<AspectRatioIcon sizeIcon={getCurrentAspectRatioIcon()} />
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
|
||||
<Tooltip
|
||||
content={
|
||||
useSystemPrompt
|
||||
? "Use system prompt: ON"
|
||||
: "Use system prompt: OFF"
|
||||
}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={useSystemPrompt}
|
||||
icon={faMessageXmark}
|
||||
activeIcon={faMessageCheck}
|
||||
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
|
||||
/>
|
||||
</Tooltip>
|
||||
{resolutionPickerOptions && (
|
||||
<Tooltip
|
||||
content="Resolution"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick={true}
|
||||
>
|
||||
<PopoverMenu
|
||||
items={resolutionPickerOptions}
|
||||
onSelect={handleResolutionSelect}
|
||||
mode="toggle"
|
||||
panelTitle="Resolution"
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
{durationOptions && (
|
||||
<Tooltip
|
||||
content="Duration"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick={true}
|
||||
>
|
||||
<PopoverMenu
|
||||
items={durationOptions}
|
||||
onSelect={handleDurationSelect}
|
||||
mode="toggle"
|
||||
panelTitle="Duration"
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon icon={faClock} className="h-3.5 w-3.5" />
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
{selectedModel?.supportsSystemPrompt !== false && (
|
||||
<Tooltip
|
||||
content={
|
||||
useSystemPrompt
|
||||
? "Use system prompt: ON"
|
||||
: "Use system prompt: OFF"
|
||||
}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={useSystemPrompt}
|
||||
icon={faMessageXmark}
|
||||
activeIcon={faMessageCheck}
|
||||
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
{selectedModel?.generateWithSound && (
|
||||
<Tooltip
|
||||
@@ -517,6 +803,22 @@ export const PromptBoxVideo = ({
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
|
||||
{inputModeOptions && (
|
||||
<Tooltip
|
||||
content="Input Mode"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick={true}
|
||||
>
|
||||
<PopoverMenu
|
||||
items={inputModeOptions}
|
||||
onSelect={handleInputModeSelect}
|
||||
mode="toggle"
|
||||
panelTitle="Input Mode"
|
||||
/>
|
||||
</Tooltip>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{modelNeedsAnImageButNoneAreSelected && (
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import { CommonAspectRatio } from "@storyteller/model-list";
|
||||
import { SizeIconOption } from "@storyteller/model-list";
|
||||
|
||||
/**
|
||||
* Renders a proportionally-sized rounded rectangle SVG icon
|
||||
* that visually represents the actual shape of an aspect ratio.
|
||||
*
|
||||
* Supports three input modes:
|
||||
* - `ratio` — raw [width, height] tuple
|
||||
* - `commonAspectRatio` — CommonAspectRatio enum (image models)
|
||||
* - `sizeIcon` — SizeIconOption enum (video models)
|
||||
*/
|
||||
export const AspectRatioIcon = ({
|
||||
ratio,
|
||||
commonAspectRatio,
|
||||
sizeIcon,
|
||||
size = 16,
|
||||
}: {
|
||||
ratio?: [number, number];
|
||||
commonAspectRatio?: CommonAspectRatio;
|
||||
sizeIcon?: SizeIconOption;
|
||||
size?: number;
|
||||
}) => {
|
||||
let rw: number;
|
||||
let rh: number;
|
||||
|
||||
if (ratio) {
|
||||
[rw, rh] = ratio;
|
||||
} else if (commonAspectRatio !== undefined) {
|
||||
[rw, rh] = commonAspectRatioToProportions(commonAspectRatio);
|
||||
} else if (sizeIcon !== undefined) {
|
||||
[rw, rh] = sizeIconToProportions(sizeIcon);
|
||||
} else {
|
||||
[rw, rh] = [16, 10];
|
||||
}
|
||||
|
||||
const scale = (size - 2) / Math.max(rw, rh);
|
||||
const w = Math.round(rw * scale);
|
||||
const h = Math.round(rh * scale);
|
||||
const x = (size - w) / 2;
|
||||
const y = (size - h) / 2;
|
||||
|
||||
return (
|
||||
<svg
|
||||
width={size}
|
||||
height={size}
|
||||
viewBox={`0 0 ${size} ${size}`}
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<rect
|
||||
x={x}
|
||||
y={y}
|
||||
width={w}
|
||||
height={h}
|
||||
rx={1.5}
|
||||
stroke="currentColor"
|
||||
strokeWidth={1.5}
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
};
|
||||
|
||||
// "Auto" icon — magic wand sparkle, kept as SVG to avoid FontAwesome dependency
|
||||
export const AutoIcon = ({ size = 16 }: { size?: number }) => (
|
||||
<svg
|
||||
width={size}
|
||||
height={size}
|
||||
viewBox="0 0 16 16"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<path
|
||||
d="M8 1l1.2 3.3L12.5 5.5l-3.3 1.2L8 10l-1.2-3.3L3.5 5.5l3.3-1.2L8 1z"
|
||||
stroke="currentColor"
|
||||
strokeWidth={1.2}
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
<path
|
||||
d="M12 9l.6 1.7 1.7.6-1.7.6-.6 1.7-.6-1.7L9.7 11.3l1.7-.6L12 9z"
|
||||
stroke="currentColor"
|
||||
strokeWidth={1}
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
function commonAspectRatioToProportions(
|
||||
ratio: CommonAspectRatio,
|
||||
): [number, number] {
|
||||
switch (ratio) {
|
||||
// Square
|
||||
case CommonAspectRatio.Square:
|
||||
case CommonAspectRatio.SquareHd:
|
||||
return [1, 1];
|
||||
|
||||
// Wide
|
||||
case CommonAspectRatio.Wide:
|
||||
return [16, 10];
|
||||
case CommonAspectRatio.WideFiveByFour:
|
||||
return [5, 4];
|
||||
case CommonAspectRatio.WideFourByThree:
|
||||
return [4, 3];
|
||||
case CommonAspectRatio.WideThreeByTwo:
|
||||
return [3, 2];
|
||||
case CommonAspectRatio.WideSixteenByNine:
|
||||
return [16, 9];
|
||||
case CommonAspectRatio.WideTwentyOneByNine:
|
||||
return [21, 9];
|
||||
|
||||
// Tall
|
||||
case CommonAspectRatio.Tall:
|
||||
return [10, 16];
|
||||
case CommonAspectRatio.TallFourByFive:
|
||||
return [4, 5];
|
||||
case CommonAspectRatio.TallThreeByFour:
|
||||
return [3, 4];
|
||||
case CommonAspectRatio.TallTwoByThree:
|
||||
return [2, 3];
|
||||
case CommonAspectRatio.TallNineBySixteen:
|
||||
return [9, 16];
|
||||
case CommonAspectRatio.TallNineByTwentyOne:
|
||||
return [9, 21];
|
||||
|
||||
// Auto
|
||||
case CommonAspectRatio.Auto:
|
||||
case CommonAspectRatio.Auto2k:
|
||||
case CommonAspectRatio.Auto4k:
|
||||
return [1, 1]; // fallback, Auto uses the sparkle icon instead
|
||||
|
||||
default:
|
||||
return [1, 1];
|
||||
}
|
||||
}
|
||||
|
||||
function sizeIconToProportions(icon: SizeIconOption): [number, number] {
|
||||
switch (icon) {
|
||||
case SizeIconOption.Landscape16x9:
|
||||
return [16, 9];
|
||||
case SizeIconOption.Landscape:
|
||||
return [16, 10];
|
||||
case SizeIconOption.Standard4x3:
|
||||
return [4, 3];
|
||||
case SizeIconOption.Square:
|
||||
return [1, 1];
|
||||
case SizeIconOption.Portrait3x4:
|
||||
return [3, 4];
|
||||
case SizeIconOption.Portrait:
|
||||
return [10, 16];
|
||||
case SizeIconOption.Portrait9x16:
|
||||
return [9, 16];
|
||||
default:
|
||||
return [16, 10];
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,23 @@
|
||||
import { faMagicWandSparkles, faRectangle, faRectangleVertical, faRectangleWide, faSquare, IconDefinition } from "@fortawesome/pro-regular-svg-icons"
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"
|
||||
import { PopoverItem, PopoverMenu } from "@storyteller/ui-popover"
|
||||
import { Tooltip } from "@storyteller/ui-tooltip"
|
||||
import { ImageModel } from "@storyteller/model-list"
|
||||
import { CommonAspectRatio } from "@storyteller/model-list"
|
||||
|
||||
import { CommonAspectRatio } from "@storyteller/model-list";
|
||||
import { PopoverItem, PopoverMenu } from "@storyteller/ui-popover";
|
||||
import { Tooltip } from "@storyteller/ui-tooltip";
|
||||
import { ImageModel } from "@storyteller/model-list";
|
||||
import { AspectRatioIcon, AutoIcon } from "./AspectRatioIcon";
|
||||
|
||||
interface AspectRatioPickerProps {
|
||||
model: ImageModel,
|
||||
currentAspectRatio?: CommonAspectRatio,
|
||||
model: ImageModel;
|
||||
currentAspectRatio?: CommonAspectRatio;
|
||||
handleCommonAspectRatioSelect: (selected: CommonAspectRatio) => void;
|
||||
//model: ImageModel | VideoModel
|
||||
}
|
||||
|
||||
/**
|
||||
* Stateless component.
|
||||
*
|
||||
* Picker for "common aspect ratios", the new data structure Tauri accepts for
|
||||
* all image and video models (Note: this is not fully rolled out yet. Some/most
|
||||
*
|
||||
* Picker for "common aspect ratios", the new data structure Tauri accepts for
|
||||
* all image and video models (Note: this is not fully rolled out yet. Some/most
|
||||
* models may still use the old format.)
|
||||
*
|
||||
*
|
||||
* @param model - currently selected model
|
||||
* @param currentAspectRatio - currently selected aspect ratio
|
||||
* @param handleCommonAspectRatioSelect - callback when an aspect ratio is selected
|
||||
@@ -29,32 +27,38 @@ export const AspectRatioPicker = ({
|
||||
currentAspectRatio,
|
||||
handleCommonAspectRatioSelect,
|
||||
}: AspectRatioPickerProps) => {
|
||||
const useAspectRatio =
|
||||
currentAspectRatio ?? model.defaultAspectRatio ?? undefined;
|
||||
|
||||
const useAspectRatio = currentAspectRatio ?? model.defaultAspectRatio ?? undefined;
|
||||
|
||||
console.log("aspect - currentAspectRatio:", currentAspectRatio);
|
||||
console.log("aspect - useAspectRatio:", useAspectRatio);
|
||||
|
||||
const getCurrentResolutionIcon = () : IconDefinition => {
|
||||
if (!useAspectRatio) {
|
||||
return faSquare;
|
||||
}
|
||||
return getAspectRatioIcon(useAspectRatio);
|
||||
}
|
||||
const isAutoRatio =
|
||||
useAspectRatio === CommonAspectRatio.Auto ||
|
||||
useAspectRatio === CommonAspectRatio.Auto2k ||
|
||||
useAspectRatio === CommonAspectRatio.Auto4k;
|
||||
|
||||
const handleSelectAdapter = (item: PopoverItem) => {
|
||||
const ratio = popOverLabelToAspectRatio(item.label, model);
|
||||
handleCommonAspectRatioSelect(ratio);
|
||||
}
|
||||
};
|
||||
|
||||
let aspectRatioList : PopoverItem[] = [];
|
||||
const isAutoAspectRatio = (ratio: CommonAspectRatio): boolean => {
|
||||
return (
|
||||
ratio === CommonAspectRatio.Auto ||
|
||||
ratio === CommonAspectRatio.Auto2k ||
|
||||
ratio === CommonAspectRatio.Auto4k
|
||||
);
|
||||
};
|
||||
|
||||
let aspectRatioList: PopoverItem[] = [];
|
||||
|
||||
model.aspectRatios?.forEach((ratio: CommonAspectRatio) => {
|
||||
aspectRatioList.push({
|
||||
label: getAspectRatioTextLabel(ratio),
|
||||
selected: useAspectRatio === ratio,
|
||||
description: `foo ${ratio}`,
|
||||
icon: <FontAwesomeIcon icon={getAspectRatioIcon(ratio)} className="h-4 w-4" />,
|
||||
icon: isAutoAspectRatio(ratio) ? (
|
||||
<AutoIcon />
|
||||
) : (
|
||||
<AspectRatioIcon commonAspectRatio={ratio} />
|
||||
),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -73,60 +77,19 @@ export const AspectRatioPicker = ({
|
||||
panelTitle="Aspect Ratio"
|
||||
showIconsInList
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon
|
||||
icon={getCurrentResolutionIcon()}
|
||||
className="h-4 w-4"
|
||||
/>
|
||||
isAutoRatio || !useAspectRatio ? (
|
||||
<AutoIcon />
|
||||
) : (
|
||||
<AspectRatioIcon commonAspectRatio={useAspectRatio} />
|
||||
)
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
</>
|
||||
)
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
const getAspectRatioIcon = (aspectRatio: CommonAspectRatio) : IconDefinition => {
|
||||
switch (aspectRatio) {
|
||||
case CommonAspectRatio.Auto:
|
||||
return faMagicWandSparkles;
|
||||
case CommonAspectRatio.Square:
|
||||
return faSquare;
|
||||
|
||||
// Wide
|
||||
case CommonAspectRatio.Wide:
|
||||
case CommonAspectRatio.WideFiveByFour:
|
||||
case CommonAspectRatio.WideFourByThree:
|
||||
case CommonAspectRatio.WideThreeByTwo:
|
||||
return faRectangle;
|
||||
case CommonAspectRatio.WideSixteenByNine:
|
||||
case CommonAspectRatio.WideTwentyOneByNine:
|
||||
return faRectangleWide;
|
||||
|
||||
// Tall
|
||||
case CommonAspectRatio.Tall:
|
||||
case CommonAspectRatio.TallFourByFive:
|
||||
case CommonAspectRatio.TallThreeByFour:
|
||||
case CommonAspectRatio.TallTwoByThree:
|
||||
return faRectangleVertical;
|
||||
case CommonAspectRatio.TallNineBySixteen:
|
||||
case CommonAspectRatio.TallNineByTwentyOne:
|
||||
return faRectangleVertical; // TODO: New FontAwesome 7 has "Tall".
|
||||
|
||||
// With resolution baked in
|
||||
case CommonAspectRatio.Auto2k:
|
||||
return faMagicWandSparkles;
|
||||
case CommonAspectRatio.Auto4k:
|
||||
return faMagicWandSparkles;
|
||||
case CommonAspectRatio.SquareHd:
|
||||
return faSquare;
|
||||
|
||||
default:
|
||||
console.error("Unknown aspect ratio in icon mapping:", aspectRatio);
|
||||
return faSquare; // Fail open-ish
|
||||
}
|
||||
}
|
||||
|
||||
const getAspectRatioTextLabel = (aspectRatio: CommonAspectRatio) : string => {
|
||||
const getAspectRatioTextLabel = (aspectRatio: CommonAspectRatio): string => {
|
||||
switch (aspectRatio) {
|
||||
case CommonAspectRatio.Auto:
|
||||
return "Auto";
|
||||
@@ -175,28 +138,46 @@ const getAspectRatioTextLabel = (aspectRatio: CommonAspectRatio) : string => {
|
||||
console.error("Unknown aspect ratio:", aspectRatio);
|
||||
return "Square"; // Fail open-ish
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Note: We only need this to deal with turning PopOverItems back into typesafe aspect ratios
|
||||
const popOverLabelToAspectRatio = (label: string, model: ImageModel): CommonAspectRatio => {
|
||||
const popOverLabelToAspectRatio = (
|
||||
label: string,
|
||||
model: ImageModel,
|
||||
): CommonAspectRatio => {
|
||||
switch (label) {
|
||||
case "Auto": return CommonAspectRatio.Auto;
|
||||
case "Square": return CommonAspectRatio.Square;
|
||||
case "5:4 (Wide)": return CommonAspectRatio.WideFiveByFour;
|
||||
case "4:3 (Wide)": return CommonAspectRatio.WideFourByThree;
|
||||
case "3:2 (Wide)": return CommonAspectRatio.WideThreeByTwo;
|
||||
case "16:9 (Wide)": return CommonAspectRatio.WideSixteenByNine;
|
||||
case "21:9 (Wide)": return CommonAspectRatio.WideTwentyOneByNine;
|
||||
case "4:5 (Tall)": return CommonAspectRatio.TallFourByFive;
|
||||
case "3:4 (Tall)": return CommonAspectRatio.TallThreeByFour;
|
||||
case "2:3 (Tall)": return CommonAspectRatio.TallTwoByThree;
|
||||
case "9:16 (Tall)": return CommonAspectRatio.TallNineBySixteen;
|
||||
case "9:21 (Tall)": return CommonAspectRatio.TallNineByTwentyOne;
|
||||
case "Auto (2K)": return CommonAspectRatio.Auto2k;
|
||||
case "Auto (4K)": return CommonAspectRatio.Auto4k;
|
||||
case "Square (HD)": return CommonAspectRatio.SquareHd;
|
||||
case "Auto":
|
||||
return CommonAspectRatio.Auto;
|
||||
case "Square":
|
||||
return CommonAspectRatio.Square;
|
||||
case "5:4 (Wide)":
|
||||
return CommonAspectRatio.WideFiveByFour;
|
||||
case "4:3 (Wide)":
|
||||
return CommonAspectRatio.WideFourByThree;
|
||||
case "3:2 (Wide)":
|
||||
return CommonAspectRatio.WideThreeByTwo;
|
||||
case "16:9 (Wide)":
|
||||
return CommonAspectRatio.WideSixteenByNine;
|
||||
case "21:9 (Wide)":
|
||||
return CommonAspectRatio.WideTwentyOneByNine;
|
||||
case "4:5 (Tall)":
|
||||
return CommonAspectRatio.TallFourByFive;
|
||||
case "3:4 (Tall)":
|
||||
return CommonAspectRatio.TallThreeByFour;
|
||||
case "2:3 (Tall)":
|
||||
return CommonAspectRatio.TallTwoByThree;
|
||||
case "9:16 (Tall)":
|
||||
return CommonAspectRatio.TallNineBySixteen;
|
||||
case "9:21 (Tall)":
|
||||
return CommonAspectRatio.TallNineByTwentyOne;
|
||||
case "Auto (2K)":
|
||||
return CommonAspectRatio.Auto2k;
|
||||
case "Auto (4K)":
|
||||
return CommonAspectRatio.Auto4k;
|
||||
case "Square (HD)":
|
||||
return CommonAspectRatio.SquareHd;
|
||||
}
|
||||
console.error("Unknown aspect ratio label:", label, "for model:", model.id);
|
||||
// If we can't find it, return the model's default aspect ratio or Square as fallback
|
||||
return model.defaultAspectRatio || CommonAspectRatio.Square;
|
||||
}
|
||||
};
|
||||
@@ -98,34 +98,48 @@ export const usePromptImageStore = create<PromptImageStore>()((set) => ({
|
||||
}));
|
||||
|
||||
// ----- Video Prompt Box Store -----
|
||||
export type VideoInputMode = "keyframe" | "reference";
|
||||
|
||||
interface PromptVideoStore {
|
||||
prompt: string;
|
||||
resolution: Resolution | string;
|
||||
aspectRatio: string | null;
|
||||
useSystemPrompt: boolean;
|
||||
referenceImages: RefImage[];
|
||||
endFrameImage?: RefImage;
|
||||
generateWithSound: boolean;
|
||||
duration: number | null;
|
||||
inputMode: VideoInputMode;
|
||||
setPrompt: (prompt: string) => void;
|
||||
setResolution: (resolution: Resolution | string) => void;
|
||||
setAspectRatio: (aspectRatio: string | null) => void;
|
||||
setUseSystemPrompt: (value: boolean) => void;
|
||||
setReferenceImages: (images: RefImage[]) => void;
|
||||
setEndFrameImage: (image?: RefImage) => void;
|
||||
setGenerateWithSound: (value: boolean) => void;
|
||||
setDuration: (duration: number | null) => void;
|
||||
setInputMode: (mode: VideoInputMode) => void;
|
||||
}
|
||||
|
||||
export const usePromptVideoStore = create<PromptVideoStore>()((set) => ({
|
||||
prompt: "",
|
||||
resolution: "720p",
|
||||
aspectRatio: null,
|
||||
useSystemPrompt: true,
|
||||
referenceImages: [],
|
||||
endFrameImage: undefined,
|
||||
generateWithSound: true,
|
||||
duration: null,
|
||||
inputMode: "keyframe",
|
||||
setPrompt: (prompt) => set({ prompt }),
|
||||
setResolution: (resolution) => set({ resolution }),
|
||||
setAspectRatio: (aspectRatio) => set({ aspectRatio }),
|
||||
setUseSystemPrompt: (useSystemPrompt) => set({ useSystemPrompt }),
|
||||
setReferenceImages: (referenceImages) => set({ referenceImages }),
|
||||
setEndFrameImage: (endFrameImage) => set({ endFrameImage }),
|
||||
setGenerateWithSound: (generateWithSound) => set({ generateWithSound }),
|
||||
setDuration: (duration) => set({ duration }),
|
||||
setInputMode: (inputMode) => set({ inputMode }),
|
||||
}));
|
||||
|
||||
// ----- Edit Prompt Box Store -----
|
||||
|
||||
@@ -26,6 +26,27 @@ export class VideoModel extends Model {
|
||||
// Whether this model supports toggling generation with sound
|
||||
readonly generateWithSound?: boolean;
|
||||
|
||||
// Available duration options in seconds (e.g. [4, 5, 6, 7, 8, 9])
|
||||
readonly durationOptions?: number[];
|
||||
|
||||
// Default duration in seconds
|
||||
readonly defaultDuration?: number;
|
||||
|
||||
// Whether the model supports multi-image reference mode
|
||||
readonly supportsReferenceMode?: boolean;
|
||||
|
||||
// Maximum number of reference images in reference mode
|
||||
readonly maxReferenceImages?: number;
|
||||
|
||||
// Available resolution options (e.g. ["480p", "720p"])
|
||||
readonly resolutionOptions?: string[];
|
||||
|
||||
// Default resolution
|
||||
readonly defaultResolution?: string;
|
||||
|
||||
// Whether the model supports the system prompt toggle (default true)
|
||||
readonly supportsSystemPrompt: boolean;
|
||||
|
||||
constructor(args: {
|
||||
id: string;
|
||||
tauriId: string;
|
||||
@@ -43,6 +64,13 @@ export class VideoModel extends Model {
|
||||
progressBarTime?: number;
|
||||
generateWithSound?: boolean;
|
||||
providers?: GenerationProvider[];
|
||||
durationOptions?: number[];
|
||||
defaultDuration?: number;
|
||||
supportsReferenceMode?: boolean;
|
||||
maxReferenceImages?: number;
|
||||
resolutionOptions?: string[];
|
||||
defaultResolution?: string;
|
||||
supportsSystemPrompt?: boolean;
|
||||
}) {
|
||||
super(args);
|
||||
this.startFrame = args.startFrame;
|
||||
@@ -50,5 +78,12 @@ export class VideoModel extends Model {
|
||||
this.requiresImage = args.requiresImage;
|
||||
this.sizeOptions = args.sizeOptions ?? [];
|
||||
this.generateWithSound = args.generateWithSound || false;
|
||||
this.durationOptions = args.durationOptions;
|
||||
this.defaultDuration = args.defaultDuration;
|
||||
this.supportsReferenceMode = args.supportsReferenceMode;
|
||||
this.maxReferenceImages = args.maxReferenceImages;
|
||||
this.resolutionOptions = args.resolutionOptions;
|
||||
this.defaultResolution = args.defaultResolution;
|
||||
this.supportsSystemPrompt = args.supportsSystemPrompt ?? true;
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ export const MODEL_TYPE_TO_CREATOR: Record<string, ModelCreator> = {
|
||||
kling_2p1_pro: ModelCreator.Kling,
|
||||
kling_2p1_master: ModelCreator.Kling,
|
||||
seedance_1p0_lite: ModelCreator.Bytedance,
|
||||
seedance_2p0: ModelCreator.Bytedance,
|
||||
sora_2: ModelCreator.OpenAi,
|
||||
veo_2: ModelCreator.Google,
|
||||
gemini_25_flash: ModelCreator.Google,
|
||||
@@ -117,6 +118,7 @@ export const getModelDisplayName = (modelType: string): string => {
|
||||
kling_2p1_pro: "Kling 2.1 Pro",
|
||||
kling_2p1_master: "Kling 2.1 Master",
|
||||
seedance_1p0_lite: "Seedance 1.0 Lite",
|
||||
seedance_2p0: "Seedance 2.0",
|
||||
veo_2: "Veo 2",
|
||||
sora_2: "Sora 2",
|
||||
recraft_3: "Recraft 3",
|
||||
|
||||
@@ -3,6 +3,10 @@ export enum SizeIconOption {
|
||||
Landscape,
|
||||
Portrait,
|
||||
Square,
|
||||
Landscape16x9,
|
||||
Portrait9x16,
|
||||
Standard4x3,
|
||||
Portrait3x4,
|
||||
}
|
||||
|
||||
export interface SizeOption {
|
||||
|
||||
@@ -10,9 +10,7 @@ export const VIDEO_MODELS: VideoModel[] = [
|
||||
fullName: "Grok Video",
|
||||
category: "video",
|
||||
creator: ModelCreator.Grok,
|
||||
providers: [
|
||||
GenerationProvider.Grok
|
||||
],
|
||||
providers: [GenerationProvider.Grok],
|
||||
selectorName: "Grok Video",
|
||||
selectorDescription: "Fastest video model",
|
||||
selectorBadges: ["20 sec."],
|
||||
@@ -141,22 +139,39 @@ export const VIDEO_MODELS: VideoModel[] = [
|
||||
requiresImage: false,
|
||||
sizeOptions: [
|
||||
{
|
||||
tauriValue: "landscape",
|
||||
textLabel: "Landscape",
|
||||
icon: SizeIconOption.Landscape,
|
||||
},
|
||||
{
|
||||
tauriValue: "portrait",
|
||||
textLabel: "Portrait",
|
||||
icon: SizeIconOption.Portrait,
|
||||
tauriValue: "wide_sixteen_by_nine",
|
||||
textLabel: "16:9",
|
||||
icon: SizeIconOption.Landscape16x9,
|
||||
},
|
||||
{
|
||||
tauriValue: "square",
|
||||
textLabel: "Square",
|
||||
textLabel: "1:1",
|
||||
icon: SizeIconOption.Square,
|
||||
},
|
||||
{
|
||||
tauriValue: "tall_three_by_four",
|
||||
textLabel: "3:4",
|
||||
icon: SizeIconOption.Portrait3x4,
|
||||
},
|
||||
{
|
||||
tauriValue: "wide_four_by_three",
|
||||
textLabel: "4:3",
|
||||
icon: SizeIconOption.Standard4x3,
|
||||
},
|
||||
{
|
||||
tauriValue: "tall_nine_by_sixteen",
|
||||
textLabel: "9:16",
|
||||
icon: SizeIconOption.Portrait9x16,
|
||||
},
|
||||
],
|
||||
progressBarTime: 100000,
|
||||
durationOptions: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
||||
defaultDuration: 5,
|
||||
supportsReferenceMode: true,
|
||||
maxReferenceImages: 3,
|
||||
resolutionOptions: ["480p", "720p"],
|
||||
defaultResolution: "720p",
|
||||
supportsSystemPrompt: false,
|
||||
progressBarTime: 360000,
|
||||
}),
|
||||
new VideoModel({
|
||||
id: "sora_2",
|
||||
@@ -164,10 +179,7 @@ export const VIDEO_MODELS: VideoModel[] = [
|
||||
fullName: "Sora 2",
|
||||
category: "video",
|
||||
creator: ModelCreator.OpenAi,
|
||||
providers: [
|
||||
GenerationProvider.Artcraft,
|
||||
GenerationProvider.Sora,
|
||||
],
|
||||
providers: [GenerationProvider.Artcraft, GenerationProvider.Sora],
|
||||
selectorName: "Sora 2",
|
||||
selectorDescription: "Smart video model",
|
||||
selectorBadges: ["2 min."],
|
||||
|
||||
@@ -44,6 +44,16 @@ export interface EnqueueImageToVideoRequest {
|
||||
|
||||
// Optional. Whether to generate audio alongside the video (used by some models like Veo2)
|
||||
generate_audio?: boolean;
|
||||
|
||||
// Optional. Common aspect ratio for other models.
|
||||
// Not all are supported by all models, but Artcraft will compensate.
|
||||
aspect_ratio?: string; // TODO: Typesafety.
|
||||
|
||||
// Optional. Duration in seconds.
|
||||
duration_seconds?: number;
|
||||
|
||||
// Optional. Reference image media tokens (for reference mode).
|
||||
reference_image_media_tokens?: string[];
|
||||
}
|
||||
|
||||
interface RawEnqueueImageToVideoRequest {
|
||||
@@ -55,8 +65,11 @@ interface RawEnqueueImageToVideoRequest {
|
||||
frontend_caller?: string;
|
||||
frontend_subscriber_id?: string;
|
||||
sora_orientation?: "portrait" | "landscape";
|
||||
grok_aspect_ratio?: "portrait" | "landscape" | "square";
|
||||
grok_aspect_ratio?: "portrait" | "landscape" | "square";
|
||||
generate_audio?: boolean;
|
||||
aspect_ratio?: string; // TODO: Typesafety.
|
||||
duration_seconds?: number;
|
||||
reference_image_media_tokens?: string[];
|
||||
}
|
||||
|
||||
export interface EnqueueImageToVideoError extends CommandResult {
|
||||
@@ -75,7 +88,7 @@ export type EnqueueImageToVideoResult =
|
||||
| EnqueueImageToVideoError;
|
||||
|
||||
export const EnqueueImageToVideo = async (
|
||||
request: EnqueueImageToVideoRequest
|
||||
request: EnqueueImageToVideoRequest,
|
||||
): Promise<EnqueueImageToVideoResult> => {
|
||||
const mutableRequest: RawEnqueueImageToVideoRequest = {
|
||||
model: request.model?.tauriId,
|
||||
@@ -115,6 +128,22 @@ export const EnqueueImageToVideo = async (
|
||||
mutableRequest.generate_audio = request.generate_audio;
|
||||
}
|
||||
|
||||
if (request.aspect_ratio) {
|
||||
mutableRequest.aspect_ratio = request.aspect_ratio;
|
||||
}
|
||||
|
||||
if (typeof request.duration_seconds === "number") {
|
||||
mutableRequest.duration_seconds = request.duration_seconds;
|
||||
}
|
||||
|
||||
if (
|
||||
request.reference_image_media_tokens &&
|
||||
request.reference_image_media_tokens.length > 0
|
||||
) {
|
||||
mutableRequest.reference_image_media_tokens =
|
||||
request.reference_image_media_tokens;
|
||||
}
|
||||
|
||||
const result = await invoke("enqueue_image_to_video_command", {
|
||||
request: mutableRequest,
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user