artcraft: migrate frontend image/video calls to unified generate_* tauri commands

Add GenerateImage / GenerateVideo wrappers in @storyteller/tauri-api
that mirror the unified TauriGenerateImageRequest / TauriGenerateVideoRequest
shapes and invoke generate_image_command / generate_video_command. Migrate
PromptBoxImage (text-to-image), PromptBoxVideo (image-to-video), PageDraw
(edit + inpaint), and Angles off the per-flow Enqueue* wrappers onto the
unified ones. Old wrappers stay; Rust proxy handles routing.
This commit is contained in:
Tanish Manku authored and Brandon Thomas committed 2026-05-03 00:28:58 -04:00
1 parent cf091fbc4d
commit 94846b580b
7 files changed
+398 -78

No files matched your search

@@ -30,7 +30,7 @@ import { twMerge } from "tailwind-merge";
import { LoadingSpinner } from "@storyteller/ui-loading-spinner";
import { SliderV2 } from "@storyteller/ui-sliderv2";
// import { Switch } from "@headlessui/react";
import { EnqueueEditImage } from "@storyteller/tauri-api";
import { GenerateImage } from "@storyteller/tauri-api";
import {
ClassyModelSelector,
ANGLES_PAGE_MODEL_LIST,
@@ -278,14 +278,14 @@ export const Angles = () => {
});
try {
await EnqueueEditImage({
model: selectedModel ?? ("flux_2_lora_angles" as any),
await GenerateImage({
model: selectedModel ?? "flux_2_lora_angles",
provider: selectedProvider ?? undefined,
image_media_tokens: [state.sourceMediaToken],
prompt: "",
horizontal_angle: state.angleConfig.rotation,
vertical_angle: state.angleConfig.tilt,
zoom: state.angleConfig.zoom,
adjust_horizontal_angle: state.angleConfig.rotation,
adjust_vertical_angle: state.angleConfig.tilt,
adjust_zoom: state.angleConfig.zoom,
frontend_subscriber_id: subscriberId,
});
@@ -7,35 +7,32 @@ import {
type ImageBundle,
} from "@storyteller/ui-pagedraw";
import {
EnqueueEditImage,
EnqueueEditImageRequest,
EnqueueEditImageSize,
EnqueueEditImageResolution,
EnqueueImageInpaint,
EnqueueImageInpaintRequest,
GenerateImage,
GenerateImageRequest,
EnqueueImageBgRemoval,
useCanvasBgRemovedEvent,
} from "@storyteller/tauri-api";
import { CommonAspectRatio, CommonResolution } from "@storyteller/model-list";
import { useImageEditCompleteEvent } from "@storyteller/tauri-events";
import { UploadImageMedia } from "@storyteller/api";
import { BaseImageSelector } from "./BaseImageSelector";
// ─── Aspect ratio / resolution mappers ────────────────────────────────────────
const mapAspectRatio = (ratio?: string): EnqueueEditImageSize | undefined => {
const mapAspectRatio = (ratio?: string): CommonAspectRatio | undefined => {
switch (ratio) {
case "auto": return EnqueueEditImageSize.Auto;
case "wide": return EnqueueEditImageSize.Wide;
case "tall": return EnqueueEditImageSize.Tall;
case "square": return EnqueueEditImageSize.Square;
case "auto": return CommonAspectRatio.Auto;
case "wide": return CommonAspectRatio.Wide;
case "tall": return CommonAspectRatio.Tall;
case "square": return CommonAspectRatio.Square;
default: return undefined;
}
};
const mapResolution = (res?: string): EnqueueEditImageResolution | undefined => {
const mapResolution = (res?: string): CommonResolution | undefined => {
switch (res) {
case "1k": return EnqueueEditImageResolution.OneK;
case "2k": return EnqueueEditImageResolution.TwoK;
case "4k": return EnqueueEditImageResolution.FourK;
case "1k": return CommonResolution.OneK;
case "2k": return CommonResolution.TwoK;
case "4k": return CommonResolution.FourK;
default: return undefined;
}
};
@@ -84,34 +81,37 @@ const useTauriAdapter = (): PageDrawAdapter => {
return useMemo<PageDrawAdapter>(
() => ({
enqueueEditImage: async (req) => {
const request: EnqueueEditImageRequest = {
const request: GenerateImageRequest = {
model: req.model,
scene_image_media_token: req.sceneImageMediaToken,
image_media_tokens: req.imageMediaTokens,
prompt: req.prompt,
disable_system_prompt: req.disableSystemPrompt,
image_count: req.imageCount,
enable_system_prompt:
typeof req.disableSystemPrompt === "boolean"
? !req.disableSystemPrompt
: undefined,
batch_size: req.imageCount,
aspect_ratio: mapAspectRatio(req.aspectRatio),
image_resolution: mapResolution(req.imageResolution),
resolution: mapResolution(req.imageResolution),
frontend_caller: req.frontendCaller,
frontend_subscriber_id: req.frontendSubscriberId,
};
if (req.provider) request.provider = req.provider;
return EnqueueEditImage(request);
return GenerateImage(request);
},
enqueueInpaint: async (req) => {
const request: EnqueueImageInpaintRequest = {
const request: GenerateImageRequest = {
model: req.model,
image_media_token: req.imageMediaToken,
mask_image_raw_bytes: req.maskImageRawBytes,
image_media_tokens: req.imageMediaToken ? [req.imageMediaToken] : undefined,
inpainting_mask_image_raw_bytes: req.maskImageRawBytes,
prompt: req.prompt,
image_count: req.imageCount,
batch_size: req.imageCount,
frontend_caller: req.frontendCaller,
frontend_subscriber_id: req.frontendSubscriberId,
};
if (req.provider) request.provider = req.provider;
return EnqueueImageInpaint(request);
return GenerateImage(request);
},
enqueueBgRemoval: async (base64Image, nodeId) => {
@@ -7,10 +7,8 @@ import { Tooltip } from "@storyteller/ui-tooltip";
import { Button, ToggleButton, GenerateButton } from "@storyteller/ui-button";
import { Modal } from "@storyteller/ui-modal";
import {
EnqueueTextToImage,
EnqueueTextToImageRequest,
EnqueueTextToImageSize,
EnqueueTextToImageResolution,
GenerateImage,
GenerateImageRequest,
} from "@storyteller/tauri-api";
import {
faMessageXmark,
@@ -296,15 +294,10 @@ export const PromptBoxImage = ({
}, 10000);
try {
const aspectRatio = getCurrentLegacyAspectRatio();
const resolution = getCurrentResolution();
const request: EnqueueTextToImageRequest = {
const request: GenerateImageRequest = {
prompt: prompt,
model: selectedModel,
aspect_ratio: aspectRatio,
image_resolution: resolution,
number_images: generationCount,
batch_size: generationCount,
frontend_caller: "text_to_image",
frontend_subscriber_id: subscriberId,
};
@@ -314,11 +307,11 @@ export const PromptBoxImage = ({
}
if (selectedModel?.supportsNewAspectRatio()) {
request.common_aspect_ratio = commonAspectRatio;
request.aspect_ratio = commonAspectRatio;
}
if (selectedModel?.supportsNewResolution()) {
request.common_resolution = commonResolution;
request.resolution = commonResolution;
}
if (selectedModel?.supportsQuality()) {
@@ -342,7 +335,7 @@ export const PromptBoxImage = ({
timestamp: Date.now(),
});
const generateResponse = await EnqueueTextToImage(request);
const generateResponse = await GenerateImage(request);
console.debug("PromptBoxImage - generateResponse", generateResponse);
await onEnqueuePressed?.(prompt, generationCount, subscriberId);
@@ -368,33 +361,6 @@ export const PromptBoxImage = ({
}
};
const getCurrentLegacyAspectRatio = (): EnqueueTextToImageSize => {
const selected = aspectRatioList.find((item) => item.selected);
switch (selected?.label.toLowerCase()) {
case "wide":
return EnqueueTextToImageSize.Wide;
case "tall":
return EnqueueTextToImageSize.Tall;
case "square":
default:
return EnqueueTextToImageSize.Square;
}
};
const getCurrentResolution = (): EnqueueTextToImageResolution | undefined => {
const selected = resolutionList.find((item) => item.selected);
switch (selected?.label) {
case "1k":
return EnqueueTextToImageResolution.OneK;
case "2k":
return EnqueueTextToImageResolution.TwoK;
case "4k":
return EnqueueTextToImageResolution.FourK;
default:
return undefined;
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key !== "Enter") return;
const isSubmitCombo = enterToGenerate && !e.shiftKey;
@@ -8,8 +8,8 @@ import { Tooltip } from "@storyteller/ui-tooltip";
import { ToggleButton, GenerateButton } from "@storyteller/ui-button";
import { Modal } from "@storyteller/ui-modal";
import {
EnqueueImageToVideo,
EnqueueImageToVideoRequest,
GenerateVideo,
GenerateVideoRequest,
} from "@storyteller/tauri-api";
import {
faWaveformLines,
@@ -703,10 +703,10 @@ export const PromptBoxVideo = ({
setIsEnqueueing(false);
}, 10000);
const buildRequest = (subscriberId: string): EnqueueImageToVideoRequest => {
let request: EnqueueImageToVideoRequest = {
const buildRequest = (subscriberId: string): GenerateVideoRequest => {
let request: GenerateVideoRequest = {
model: selectedModel,
image_media_token: imageMediaToken,
start_frame_image_media_token: imageMediaToken,
prompt: prompt,
end_frame_image_media_token: isRefMode
? undefined
@@ -816,7 +816,7 @@ export const PromptBoxVideo = ({
? crypto.randomUUID()
: Math.random().toString(36).slice(2);
subscriberIds.push(subscriberId);
enqueuePromises.push(EnqueueImageToVideo(buildRequest(subscriberId)));
enqueuePromises.push(GenerateVideo(buildRequest(subscriberId)));
}
try {
+2
View File
@@ -14,6 +14,8 @@ export * from "./lib/enqueue/EnqueueImageTo3dWorld.js";
export * from "./lib/enqueue/EnqueueImageToGaussian.js";
export * from "./lib/enqueue/EnqueueImageToVideo.js";
export * from "./lib/enqueue/EnqueueTextToImage.js";
export * from "./lib/generate/GenerateImage.js";
export * from "./lib/generate/GenerateVideo.js";
export * from "./lib/events/BasicEventWrapper.js";
export * from "./lib/events/CanvasBgRemovedEvent.js";
export * from "./lib/fal/FalBackgroundRemoval.js";
@@ -0,0 +1,185 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import {
CommonAspectRatio,
CommonResolution,
CommonQuality,
ImageModel,
} from "@storyteller/model-list";
import { GenerationProvider } from "@storyteller/api-enums";
export interface GenerateImageRequest {
// The provider to use (defaults to Artcraft/Storyteller).
provider?: GenerationProvider;
// The model to use.
model?: ImageModel | string;
// Text prompt for the image generation.
prompt?: string;
// Aspect ratio.
aspect_ratio?: CommonAspectRatio;
// Resolution.
resolution?: CommonResolution;
// Quality (used by OpenAI models).
quality?: CommonQuality;
// Number of images to generate.
batch_size?: number;
// Reference images (without semantics — purpose varies per model).
image_media_tokens?: string[];
// Canvas image — supply this XOR canvas_image_raw_bytes.
// Becomes the first image reference (pushing back image_media_tokens by one).
canvas_image_media_token?: string;
canvas_image_raw_bytes?: Uint8Array;
// Scene image — supply this XOR scene_image_raw_bytes.
scene_image_media_token?: string;
scene_image_raw_bytes?: Uint8Array;
// Inpainting mask — supply this XOR inpainting_mask_image_raw_bytes.
inpainting_mask_image_media_token?: string;
inpainting_mask_image_raw_bytes?: Uint8Array;
// Angle adjustments (for edit models like QwenEdit, Flux2LoraAngles).
adjust_horizontal_angle?: number;
adjust_vertical_angle?: number;
adjust_zoom?: number;
// Turn on the system prompt.
enable_system_prompt?: boolean;
// Frontend metadata.
frontend_caller?: string;
frontend_subscriber_id?: string;
frontend_subscriber_payload?: string;
}
interface RawGenerateImageRequest {
provider?: GenerationProvider;
model?: string;
prompt?: string;
aspect_ratio?: CommonAspectRatio;
resolution?: CommonResolution;
quality?: CommonQuality;
batch_size?: number;
image_media_tokens?: string[];
canvas_image_media_token?: string;
canvas_image_raw_bytes?: Uint8Array;
scene_image_media_token?: string;
scene_image_raw_bytes?: Uint8Array;
inpainting_mask_image_media_token?: string;
inpainting_mask_image_raw_bytes?: Uint8Array;
adjust_horizontal_angle?: number;
adjust_vertical_angle?: number;
adjust_zoom?: number;
enable_system_prompt?: boolean;
frontend_caller?: string;
frontend_subscriber_id?: string;
frontend_subscriber_payload?: string;
}
export enum GenerateImageErrorType {
ModelNotSpecified = "model_not_specified",
BadInput = "bad_input",
NoProviderAvailable = "no_provider_available",
ServerError = "server_error",
NeedsStorytellerCredentials = "needs_storyteller_credentials",
NeedsGrokCredentials = "needs_grok_credentials",
BillingIssue = "billing_issue",
}
export interface GenerateImageError extends CommandResult {
error_type: GenerateImageErrorType;
error_message?: string;
}
export type GenerateImagePayload = Record<string, never>;
export interface GenerateImageSuccess extends CommandResult {
payload: GenerateImagePayload;
}
export type GenerateImageResult = GenerateImageSuccess | GenerateImageError;
export const GenerateImage = async (
request: GenerateImageRequest,
): Promise<GenerateImageResult> => {
let modelName: string | undefined;
if (!!request.model) {
if (typeof request.model === "string") {
modelName = request.model;
} else if (typeof request.model.tauriId === "string") {
modelName = request.model.tauriId;
}
}
if (!modelName) {
throw new Error(
"No model specified in request: " + JSON.stringify(request),
);
}
const mutableRequest: RawGenerateImageRequest = {
model: modelName,
};
if (!!request.provider) mutableRequest.provider = request.provider;
if (!!request.prompt) mutableRequest.prompt = request.prompt;
if (!!request.aspect_ratio) mutableRequest.aspect_ratio = request.aspect_ratio;
if (!!request.resolution) mutableRequest.resolution = request.resolution;
if (!!request.quality) mutableRequest.quality = request.quality;
if (typeof request.batch_size === "number") mutableRequest.batch_size = request.batch_size;
if (!!request.image_media_tokens && request.image_media_tokens.length > 0) {
mutableRequest.image_media_tokens = request.image_media_tokens;
}
if (!!request.canvas_image_media_token) {
mutableRequest.canvas_image_media_token = request.canvas_image_media_token;
}
if (!!request.canvas_image_raw_bytes) {
mutableRequest.canvas_image_raw_bytes = request.canvas_image_raw_bytes;
}
if (!!request.scene_image_media_token) {
mutableRequest.scene_image_media_token = request.scene_image_media_token;
}
if (!!request.scene_image_raw_bytes) {
mutableRequest.scene_image_raw_bytes = request.scene_image_raw_bytes;
}
if (!!request.inpainting_mask_image_media_token) {
mutableRequest.inpainting_mask_image_media_token = request.inpainting_mask_image_media_token;
}
if (!!request.inpainting_mask_image_raw_bytes) {
mutableRequest.inpainting_mask_image_raw_bytes = request.inpainting_mask_image_raw_bytes;
}
if (typeof request.adjust_horizontal_angle === "number") {
mutableRequest.adjust_horizontal_angle = request.adjust_horizontal_angle;
}
if (typeof request.adjust_vertical_angle === "number") {
mutableRequest.adjust_vertical_angle = request.adjust_vertical_angle;
}
if (typeof request.adjust_zoom === "number") {
mutableRequest.adjust_zoom = request.adjust_zoom;
}
if (typeof request.enable_system_prompt === "boolean") {
mutableRequest.enable_system_prompt = request.enable_system_prompt;
}
if (!!request.frontend_caller) mutableRequest.frontend_caller = request.frontend_caller;
if (!!request.frontend_subscriber_id) {
mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id;
}
if (!!request.frontend_subscriber_payload) {
mutableRequest.frontend_subscriber_payload = request.frontend_subscriber_payload;
}
const result = await invoke("generate_image_command", {
request: mutableRequest,
});
return result as GenerateImageResult;
};
@@ -0,0 +1,167 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import {
CommonAspectRatio,
CommonResolution,
VideoModel,
} from "@storyteller/model-list";
import { GenerationProvider } from "@storyteller/api-enums";
export interface GenerateVideoRequest {
// The provider to use (defaults to Artcraft/Storyteller).
provider?: GenerationProvider;
// The model to use.
model?: VideoModel | string;
// Text prompt.
prompt?: string;
// Negative prompt.
negative_prompt?: string;
// Starting frame.
start_frame_image_media_token?: string;
// Ending frame.
end_frame_image_media_token?: string;
// Reference media tokens.
reference_image_media_tokens?: string[];
reference_video_media_tokens?: string[];
reference_audio_media_tokens?: string[];
reference_character_tokens?: string[];
aspect_ratio?: CommonAspectRatio;
resolution?: CommonResolution;
duration_seconds?: number;
generate_audio?: boolean;
video_batch_count?: number;
// Deprecated on the Rust side (still read by some legacy handlers).
sora_orientation?: "portrait" | "landscape";
grok_aspect_ratio?: "portrait" | "landscape" | "square";
// Frontend metadata.
frontend_caller?: string;
frontend_subscriber_id?: string;
frontend_subscriber_payload?: string;
}
interface RawGenerateVideoRequest {
provider?: GenerationProvider;
model?: string;
prompt?: string;
negative_prompt?: string;
start_frame_image_media_token?: string;
end_frame_image_media_token?: string;
reference_image_media_tokens?: string[];
reference_video_media_tokens?: string[];
reference_audio_media_tokens?: string[];
reference_character_tokens?: string[];
aspect_ratio?: CommonAspectRatio;
resolution?: CommonResolution;
duration_seconds?: number;
generate_audio?: boolean;
video_batch_count?: number;
sora_orientation?: "portrait" | "landscape";
grok_aspect_ratio?: "portrait" | "landscape" | "square";
frontend_caller?: string;
frontend_subscriber_id?: string;
frontend_subscriber_payload?: string;
}
export enum GenerateVideoErrorType {
ModelNotSpecified = "model_not_specified",
NoProviderAvailable = "no_provider_available",
ServerError = "server_error",
NeedsFalApiKey = "needs_fal_api_key",
FalError = "fal_error",
NeedsStorytellerCredentials = "needs_storyteller_credentials",
}
export interface GenerateVideoError extends CommandResult {
error_type: GenerateVideoErrorType;
error_message?: string;
}
export type GenerateVideoPayload = Record<string, never>;
export interface GenerateVideoSuccess extends CommandResult {
payload: GenerateVideoPayload;
}
export type GenerateVideoResult = GenerateVideoSuccess | GenerateVideoError;
export const GenerateVideo = async (
request: GenerateVideoRequest,
): Promise<GenerateVideoResult> => {
let modelName: string | undefined;
if (!!request.model) {
if (typeof request.model === "string") {
modelName = request.model;
} else if (typeof request.model.tauriId === "string") {
modelName = request.model.tauriId;
}
}
if (!modelName) {
throw new Error(
"No model specified in request: " + JSON.stringify(request),
);
}
const mutableRequest: RawGenerateVideoRequest = {
model: modelName,
};
if (!!request.provider) mutableRequest.provider = request.provider;
if (!!request.prompt) mutableRequest.prompt = request.prompt;
if (!!request.negative_prompt) mutableRequest.negative_prompt = request.negative_prompt;
if (!!request.start_frame_image_media_token) {
mutableRequest.start_frame_image_media_token = request.start_frame_image_media_token;
}
if (!!request.end_frame_image_media_token) {
mutableRequest.end_frame_image_media_token = request.end_frame_image_media_token;
}
if (!!request.reference_image_media_tokens && request.reference_image_media_tokens.length > 0) {
mutableRequest.reference_image_media_tokens = request.reference_image_media_tokens;
}
if (!!request.reference_video_media_tokens && request.reference_video_media_tokens.length > 0) {
mutableRequest.reference_video_media_tokens = request.reference_video_media_tokens;
}
if (!!request.reference_audio_media_tokens && request.reference_audio_media_tokens.length > 0) {
mutableRequest.reference_audio_media_tokens = request.reference_audio_media_tokens;
}
if (!!request.reference_character_tokens && request.reference_character_tokens.length > 0) {
mutableRequest.reference_character_tokens = request.reference_character_tokens;
}
if (!!request.aspect_ratio) mutableRequest.aspect_ratio = request.aspect_ratio;
if (!!request.resolution) mutableRequest.resolution = request.resolution;
if (typeof request.duration_seconds === "number") {
mutableRequest.duration_seconds = request.duration_seconds;
}
if (typeof request.generate_audio === "boolean") {
mutableRequest.generate_audio = request.generate_audio;
}
if (typeof request.video_batch_count === "number") {
mutableRequest.video_batch_count = request.video_batch_count;
}
if (!!request.sora_orientation) mutableRequest.sora_orientation = request.sora_orientation;
if (!!request.grok_aspect_ratio) mutableRequest.grok_aspect_ratio = request.grok_aspect_ratio;
if (!!request.frontend_caller) mutableRequest.frontend_caller = request.frontend_caller;
if (!!request.frontend_subscriber_id) {
mutableRequest.frontend_subscriber_id = request.frontend_subscriber_id;
}
if (!!request.frontend_subscriber_payload) {
mutableRequest.frontend_subscriber_payload = request.frontend_subscriber_payload;
}
const result = await invoke("generate_video_command", {
request: mutableRequest,
});
return result as GenerateVideoResult;
};