add fal image generation end-to-end (recraft, etc.) (#637)

This commit is contained in:
Brandon Thomas authored and GitHub committed 2025-05-26 21:33:21 -04:00
1 parent 4a10581e9a
commit 0f1cf93252
26 files changed
+634 -54

No files matched your search

@@ -0,0 +1,268 @@
use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker};
use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir;
use crate::core::utils::get_url_file_extension::get_url_file_extension;
use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir;
use crate::core::utils::simple_http_download::simple_http_download;
use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile;
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use fal_client::requests::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
use image::{DynamicImage, EncodableLayout, ImageReader};
use log::{debug, error, info, warn};
use mimetypes::mimetype_info::mimetype_info::MimetypeInfo;
use openai_sora_client::credentials::SoraCredentials;
use openai_sora_client::creds::credential_migration::CredentialMigrationRef;
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest};
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file;
use openai_sora_client::sora_error::SoraError;
use reqwest::Url;
use serde_derive::{Deserialize, Serialize};
use std::fs::read_to_string;
use std::io::{Cursor, Read, Write};
use std::ops::Add;
use std::path::PathBuf;
use std::time::Duration;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::api_error::ApiError::InternalServerError;
use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse};
use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file;
use storyteller_client::utils::api_host::ApiHost;
use tauri::{AppHandle, Emitter, Manager, State};
use tempfile::NamedTempFile;
use tokens::tokens::media_files::MediaFileToken;
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
#[derive(Deserialize)]
pub struct EnqueueTextToImageRequest {
/// Text prompt for the image generation. Required.
pub prompt: Option<String>,
/// The model to use.
pub model: Option<EnqueueTextToImageModel>,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum EnqueueTextToImageModel {
#[serde(rename = "flux_pro_ultra")]
FluxProUltra,
#[serde(rename = "recraft_3")]
Recraft3,
}
#[derive(Serialize)]
pub struct EnqueueTextToImageSuccessResponse {
}
impl SerializeMarker for EnqueueTextToImageSuccessResponse {}
#[derive(Serialize, Debug)]
#[serde(rename_all = "snake_case")]
pub enum EnqueueTextToImageErrorType {
/// Caller didn't specify a model
ModelNotSpecified,
/// Generic server error
ServerError,
/// No Fal API key available
NeedsFalApiKey,
/// Fal had an API error
FalError,
}
#[tauri::command]
pub async fn enqueue_text_to_image_command(
request: EnqueueTextToImageRequest,
app_data_root: State<'_, AppDataRoot>,
fal_creds_manager: State<'_, FalCredentialManager>,
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
fal_task_queue: State<'_, FalTaskQueue>,
) -> CommandResult<EnqueueTextToImageSuccessResponse, EnqueueTextToImageErrorType, ()> {
info!("enqueue_text_to_image called");
let result = handle_request(
request,
&app_data_root,
&fal_creds_manager,
&storyteller_creds_manager,
&fal_task_queue,
).await;
match result {
Err(err) => {
error!("error: {:?}", err);
//let event = SoraImageEnqueueFailureEvent {};
//let result = event.send(&app);
//if let Err(err) = result {
// error!("Failed to emit event: {:?}", err);
//}
let mut status = CommandErrorStatus::ServerError;
let mut error_type = EnqueueTextToImageErrorType::ServerError;
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
match err {
InnerError::NoModelSpecified => {
status = CommandErrorStatus::BadRequest;
error_type = EnqueueTextToImageErrorType::ModelNotSpecified;
error_message = "No model specified for image generation";
}
InnerError::NeedsFalApiKey => {
status = CommandErrorStatus::Unauthorized;
error_type = EnqueueTextToImageErrorType::NeedsFalApiKey;
error_message = "You need to set a FAL api key";
},
_ => {}, // Fall-through
}
Err(CommandErrorResponseWrapper {
status,
error_message: Some(error_message.to_string()),
error_type: Some(error_type),
error_details: None,
})
}
Ok(result) => {
//let event = SoraImageEnqueueSuccessEvent {};
//let result = event.send(&app);
//if let Err(err) = result {
// error!("Failed to emit event: {:?}", err);
//}
Ok(EnqueueTextToImageSuccessResponse {
}.into())
}
}
}
#[derive(Debug)]
enum InnerError {
NoModelSpecified,
NeedsFalApiKey,
FalError(FalErrorPlus),
AnyhowError(AnyhowError),
StorytellerApiError(ApiError),
DecodeError(DecodeError),
IoError(std::io::Error),
}
impl From<AnyhowError> for InnerError {
fn from(value: AnyhowError) -> Self {
Self::AnyhowError(value)
}
}
impl From<FalErrorPlus> for InnerError {
fn from(value: FalErrorPlus) -> Self {
Self::FalError(value)
}
}
impl From<ApiError> for InnerError {
fn from(value: ApiError) -> Self {
Self::StorytellerApiError(value)
}
}
impl From<DecodeError> for InnerError {
fn from(value: DecodeError) -> Self {
Self::DecodeError(value)
}
}
impl From<std::io::Error> for InnerError {
fn from(value: std::io::Error) -> Self {
Self::IoError(value)
}
}
pub async fn handle_request(
request: EnqueueTextToImageRequest,
app_data_root: &AppDataRoot,
fal_creds_manager: &FalCredentialManager,
storyteller_creds_manager: &StorytellerCredentialManager,
fal_task_queue: &FalTaskQueue,
) -> Result<(), InnerError> {
match request.model {
None => {
return Err(InnerError::NoModelSpecified);
}
Some(_) => {}
}
let has_credentials = fal_creds_manager
.has_apparent_api_token()
.unwrap_or(true); // NB: Failures would be lock issues
if !has_credentials {
warn!("No FAL API key found");
return Err(InnerError::NeedsFalApiKey);
}
let api_key = fal_creds_manager.get_key_required()?;
let prompt = request.prompt.as_deref().unwrap_or("");
let result = match request.model {
None => {
return Err(InnerError::NoModelSpecified);
}
Some(EnqueueTextToImageModel::FluxProUltra) => {
info!("enqueue Flux Pro Ultra text-to-image with prompt: {}", prompt);
enqueue_flux_pro_ultra_text_to_image(FluxProUltraTextToImageArgs {
prompt,
api_key: &api_key,
}).await
}
Some(EnqueueTextToImageModel::Recraft3) => {
info!("enqueue Recraft v3 text-to-image with prompt: {}", prompt);
enqueue_recraft3_text_to_image(Recraft3TextToImageArgs {
prompt,
api_key: &api_key,
}).await
}
};
match result {
Ok(enqueued) => {
info!("Successfully enqueued text to image");
if let Err(err) = fal_task_queue.insert(&enqueued) {
error!("Failed to enqueue task: {:?}", err);
return Err(InnerError::AnyhowError(anyhow!("Failed to enqueue task: {:?}", err)));
}
}
Err(err) => {
error!("Failed to enqueue text to image: {:?}", err);
return Err(InnerError::FalError(err));
}
}
Ok(())
}
@@ -0,0 +1 @@
pub mod enqueue_text_to_image_command;
@@ -1,5 +1,6 @@
pub mod app_preferences;
pub mod command_response_wrapper;
pub mod enqueue;
pub mod flip_image;
pub mod get_build_info_command;
pub mod platform_info_command;
+2
View File
@@ -5,6 +5,7 @@ use tauri::Manager;
use crate::core::commands::app_preferences::get_app_preferences_command::get_app_preferences_command;
use crate::core::commands::app_preferences::update_app_preference_command::update_app_preferences_command;
use crate::core::commands::enqueue::enqueue_text_to_image_command::enqueue_text_to_image_command;
use crate::core::commands::flip_image::flip_image;
use crate::core::commands::get_build_info_command::get_build_info_command;
use crate::core::commands::platform_info_command::platform_info_command;
@@ -160,6 +161,7 @@ pub fn run() {
.manage(storyteller_creds_manager_3)
.invoke_handler(tauri::generate_handler![
check_sora_session_command,
enqueue_text_to_image_command,
fal_background_removal_command,
fal_hunyuan_image_to_3d_command,
fal_kling_image_to_video_command,
@@ -19,7 +19,7 @@ use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
@@ -20,9 +20,8 @@ use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args};
use fal_client::requests::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
@@ -20,8 +20,8 @@ use anyhow::anyhow;
use base64::prelude::BASE64_STANDARD;
use base64::{DecodeError, Engine};
use errors::{AnyhowError, AnyhowResult};
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::requests::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::requests::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use fal_client::requests::remove_background_rembg::remove_background_rembg;
use filesys::file_read_bytes::file_read_bytes;
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
@@ -57,10 +57,15 @@ use tokens::tokens::media_files::MediaFileToken;
#[derive(Deserialize)]
pub struct FalKlingImageToVideoRequest {
/// Image media file; the image to remove the background from.
/// This must be supplied **OR** the `base64_image`.
pub image_media_token: Option<MediaFileToken>,
/// Base64-encoded image
/// This must be supplied **OR** the `image_media_token`.
pub base64_image: Option<String>,
/// Optional: Text prompt.
pub prompt: Option<String>,
}
#[derive(Serialize, Debug)]
@@ -209,6 +214,7 @@ pub async fn image_to_video(
image_path: filename,
api_key: &api_key,
duration: Kling16Duration::Default,
prompt: request.prompt.as_deref().unwrap_or(""),
}).await?;
if let Err(err) = fal_task_queue.insert(&enqueued) {
@@ -9,8 +9,8 @@ use crate::services::sora::events::sora_image_generation_failed_event::SoraImage
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use errors::AnyhowResult;
use fal_client::error::fal_error_plus::FalErrorPlus;
use fal_client::export::queue::Status;
use fal_client::fal_error_plus::FalErrorPlus;
use fal_client::model::fal_request_id::FalRequestId;
use fal_client::utils::queue_status_checker::QueueStatusChecker;
use log::{error, info, warn};
@@ -0,0 +1,20 @@
use crate::error::fal_error_plus::FalErrorPlus;
/// Better classify the `fal::FalError` into a more user-friendly `FalErrorPlus`.
pub fn classify_fal_error(err: fal::FalError) -> FalErrorPlus {
match err {
fal::FalError::RequestError(_) => FalErrorPlus::FalError(err),
fal::FalError::ImageError(_) => FalErrorPlus::FalError(err),
fal::FalError::SerializeError(_) => FalErrorPlus::FalError(err),
fal::FalError::StreamError(_) => FalErrorPlus::FalError(err),
fal::FalError::Other(ref inner) => {
if inner.contains("Invalid Key Authorization header format") {
FalErrorPlus::FalApiKeyError(inner.to_string())
} else if inner.contains("No user found for Key ID and Secret") {
FalErrorPlus::FalApiKeyError(inner.to_string())
} else {
FalErrorPlus::FalError(err)
}
}
}
}
@@ -1,3 +1,4 @@
use crate::error::classify_fal_error::classify_fal_error;
use std::error::Error;
use std::fmt::{Display, Formatter};
@@ -6,6 +7,8 @@ use std::fmt::{Display, Formatter};
pub enum FalErrorPlus {
/// An error arising in the `fal` crate.
FalError(fal::FalError),
/// The fal API key is invalid.
FalApiKeyError(String),
/// Another error we didn't handle.
AnyhowError(anyhow::Error),
/// URL parse errors.
@@ -18,6 +21,7 @@ impl Display for FalErrorPlus {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::FalError(err) => write!(f, "FalErrorPlus::FalError: {:?}", err),
Self::FalApiKeyError(reason) => write!(f, "FalErrorPlus::FalApiKeyError: {}", reason),
Self::AnyhowError(err) => write!(f, "FalErrorPlus::AnyhowError: {:?}", err),
Self::UrlParseError(err) => write!(f, "FalErrorPlus::UrlParseError: {:?}", err),
Self::UnhandledEndpoint(endpoint) => write!(f, "FalErrorPlus::UnhandledEndpoint: {:?}", endpoint),
@@ -29,7 +33,7 @@ impl Error for FalErrorPlus {}
impl From<fal::FalError> for FalErrorPlus {
fn from(err: fal::FalError) -> Self {
FalErrorPlus::FalError(err)
classify_fal_error(err)
}
}
@@ -0,0 +1,2 @@
pub mod classify_fal_error;
pub mod fal_error_plus;
+1 -1
View File
@@ -8,7 +8,7 @@ pub mod export {
}
pub mod creds;
pub mod fal_error_plus;
pub mod error;
pub mod model;
pub mod requests;
pub mod utils;
@@ -1,29 +1,46 @@
use fal::queue::Queue;
use serde::de::DeserializeOwned;
// TODO(bt,2025-05-26): A macro to define these + tests.
const FLUX_PRO_ULTRA_TEXT_TO_IMAGE: &str = "fal-ai/flux-pro/v1.1-ultra";
const HUNYUAN_3D_2_BASE: &str = "fal-ai/hunyuan3d/v2";
const KLING16_IMAGE_TO_VIDEO: &str = "fal-ai/kling-video/v1.6/pro/image-to-video";
const MINIMAX_IMAGE_TO_VIDEO: &str = "fal-ai/minimax/image-to-video";
const RECRAFT_V3_TEXT_TO_IMAGE: &str = "fal-ai/recraft-v3";
#[derive(Clone, Debug)]
pub enum FalEndpoint {
Kling16ImageToVideo,
FluxProUltraTextToImage,
Hunyuan3d2Base,
Kling16ImageToVideo,
Minimax01ImageToVideo,
RecraftV3TextToImage,
Other(String),
}
impl FalEndpoint {
pub fn from_queue_response<R: DeserializeOwned>(queue: &Queue<R>) -> Self {
match queue.endpoint.as_str() {
FLUX_PRO_ULTRA_TEXT_TO_IMAGE => FalEndpoint::FluxProUltraTextToImage,
HUNYUAN_3D_2_BASE => FalEndpoint::Hunyuan3d2Base,
KLING16_IMAGE_TO_VIDEO => FalEndpoint::Kling16ImageToVideo,
MINIMAX_IMAGE_TO_VIDEO => FalEndpoint::Minimax01ImageToVideo,
RECRAFT_V3_TEXT_TO_IMAGE => FalEndpoint::RecraftV3TextToImage,
_ => FalEndpoint::Other(queue.endpoint.clone()),
}
}
pub fn url(&self) -> &str {
match self {
Self::FluxProUltraTextToImage => FLUX_PRO_ULTRA_TEXT_TO_IMAGE,
Self::Hunyuan3d2Base => HUNYUAN_3D_2_BASE,
Self::Kling16ImageToVideo => KLING16_IMAGE_TO_VIDEO,
Self::Minimax01ImageToVideo => MINIMAX_IMAGE_TO_VIDEO,
Self::RecraftV3TextToImage => RECRAFT_V3_TEXT_TO_IMAGE,
Self::Other(url) => url,
}
}
@@ -1,5 +1,6 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::hunyuan3d::v2::v2;
use fal::endpoints::fal_ai::hunyuan3d::v2::Hunyuan3DInput;
@@ -16,12 +17,6 @@ pub struct Hunyuan2Args<'a, P: AsRef<Path>> {
pub async fn enqueue_hunyuan2_image_to_3d<P: AsRef<Path>>(args: Hunyuan2Args<'_, P>) -> Result<EnqueuedRequest, FalErrorPlus> {
let image_url = file_to_base64_url(args.image_path)?;
/*
TODO: Handle error messages -
FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected '<key_id>:<key_secret>'.\"}")))
FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}")))
*/
let request = Hunyuan3DInput {
input_image_url: image_url,
textured_mesh: Some(true),
@@ -34,7 +29,12 @@ pub async fn enqueue_hunyuan2_image_to_3d<P: AsRef<Path>>(args: Hunyuan2Args<'_,
let result = v2(request)
.with_api_key(&args.api_key.0)
.queue()
.await?;
.await;
let result = match result {
Ok(result) => result,
Err(err) => return Err(classify_fal_error(err)),
};
Ok(EnqueuedRequest::from_queue_response(&result)?)
}
@@ -0,0 +1,66 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::flux_pro::v1_1_ultra::v1_1_ultra;
use fal::endpoints::fal_ai::flux_pro::v1_1_ultra::FluxProUltraTextToImageInput;
use futures::StreamExt;
use std::io::Write;
pub struct FluxProUltraTextToImageArgs<'a> {
pub prompt: &'a str,
pub api_key: &'a FalApiKey,
// TODO: Size parameter
}
pub async fn enqueue_flux_pro_ultra_text_to_image(args: FluxProUltraTextToImageArgs<'_>) -> Result<EnqueuedRequest, FalErrorPlus> {
let request = FluxProUltraTextToImageInput {
prompt: args.prompt.to_string(),
safety_tolerance: Some("5".to_string()), // 1 is most strict, 5 is most permissive
enable_safety_checker: Some(false),
num_images: Some(1), // Default is 1
output_format: Some("png".to_string()),
raw: Some(true), // Generate less processed, more natural-looking images. Default is false.
aspect_ratio: None, // Default is "16:9"
seed: None,
sync_mode: None, // Synchronous / slow
};
let result = v1_1_ultra(request)
.with_api_key(&args.api_key.0)
.queue()
.await;
let result = match result {
Ok(result) => result,
Err(err) => return Err(classify_fal_error(err)),
};
Ok(EnqueuedRequest::from_queue_response(&result)?)
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::image_gen::enqueue_flux_pro_ultra_text_to_image::{enqueue_flux_pro_ultra_text_to_image, FluxProUltraTextToImageArgs};
use errors::AnyhowResult;
use std::fs::read_to_string;
#[tokio::test]
#[ignore]
async fn test_flux_pro() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = FluxProUltraTextToImageArgs {
prompt: "a corgi smiles at the camera, inside retro video game store, lots of sci-fi props from Cowboy Bebop, photorealistic",
api_key: &api_key,
};
let result = enqueue_flux_pro_ultra_text_to_image(args).await?;
Ok(())
}
}
@@ -0,0 +1,63 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::recraft_v3::recraft_v3;
use fal::endpoints::fal_ai::recraft_v3::ImageSizeProperty;
use fal::endpoints::fal_ai::recraft_v3::TextToImageInput;
use futures::StreamExt;
use std::io::Write;
pub struct Recraft3TextToImageArgs<'a> {
pub prompt: &'a str,
pub api_key: &'a FalApiKey,
// TODO: Size parameter
}
pub async fn enqueue_recraft3_text_to_image(args: Recraft3TextToImageArgs<'_>) -> Result<EnqueuedRequest, FalErrorPlus> {
let request = TextToImageInput {
image_size: Some(ImageSizeProperty::SquareHd),
prompt: args.prompt.to_string(),
style: None, // eg. "realistic_image"
style_id: None, // LoRA.
colors: None, // Preferable colors. We can offer this later.
};
let result = recraft_v3(request)
.with_api_key(&args.api_key.0)
.queue()
.await;
let result = match result {
Ok(result) => result,
Err(err) => return Err(classify_fal_error(err)),
};
Ok(EnqueuedRequest::from_queue_response(&result)?)
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs};
use errors::AnyhowResult;
use std::fs::read_to_string;
#[tokio::test]
#[ignore]
async fn test_recraft3() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = Recraft3TextToImageArgs {
prompt: "a corgi looks out over the water",
api_key: &api_key,
};
let result = enqueue_recraft3_text_to_image(args).await?;
Ok(())
}
}
@@ -0,0 +1,2 @@
pub mod enqueue_flux_pro_ultra_text_to_image;
pub mod enqueue_recraft3_text_to_image;
@@ -1,3 +1,4 @@
pub mod enqueue_hunyuan2_image_to_3d;
pub mod enqueue_kling_16_image_to_video;
pub mod remove_background_rembg;
pub mod remove_background_rembg;
pub mod video_gen;
pub mod image_gen;
@@ -1,5 +1,6 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::endpoints::fal_ai::imageutils::rembg::RemoveBackgroundInput;
use fal::prelude::fal_ai::imageutils::rembg::rembg;
use fal::prelude::*;
@@ -21,12 +22,6 @@ pub struct RemBgResponse {
pub async fn remove_background_rembg<P: AsRef<Path>>(image_path: P, api_key: &FalApiKey) -> Result<RemBgResponse, FalErrorPlus> {
let image_url = file_to_base64_url(image_path)?;
/*
TODO: Handle error messages -
FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected '<key_id>:<key_secret>'.\"}")))
FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}")))
*/
let request = RemoveBackgroundInput {
image_url,
crop_to_bbox: None,
@@ -36,7 +31,12 @@ pub async fn remove_background_rembg<P: AsRef<Path>>(image_path: P, api_key: &Fa
let result = rembg(request)
.with_api_key(&api_key.0)
.queue()
.await?;
.await;
let result = match result {
Ok(result) => result,
Err(err) => return Err(classify_fal_error(err)),
};
let mut stream = result.stream(true).await?;
@@ -1,15 +1,16 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::kling_video::v1_6::pro::image_to_video::{image_to_video, ProImageToVideoRequest};
use fal_client::file_to_base64_url::file_to_base64_url;
use futures::StreamExt;
use std::io::Write;
use std::path::Path;
use url::Url;
pub struct Kling16Args<'a, P: AsRef<Path>> {
pub image_path: P,
pub prompt: &'a str,
pub api_key: &'a FalApiKey,
pub duration: Kling16Duration,
}
@@ -24,12 +25,6 @@ pub enum Kling16Duration {
pub async fn enqueue_kling_16_image_to_video<P: AsRef<Path>>(args: Kling16Args<'_, P>) -> Result<EnqueuedRequest, FalErrorPlus> {
let image_url = file_to_base64_url(args.image_path)?;
/*
TODO: Handle error messages -
FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected '<key_id>:<key_secret>'.\"}")))
FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}")))
*/
let duration = match args.duration{
Kling16Duration::Default => None,
Kling16Duration::FiveSeconds => Some("5".to_string()), // Gross...
@@ -38,7 +33,7 @@ pub async fn enqueue_kling_16_image_to_video<P: AsRef<Path>>(args: Kling16Args<'
let request = ProImageToVideoRequest {
image_url,
prompt: "".to_string(),
prompt: args.prompt.to_string(),
aspect_ratio: None,
cfg_scale: None,
duration,
@@ -49,7 +44,12 @@ pub async fn enqueue_kling_16_image_to_video<P: AsRef<Path>>(args: Kling16Args<'
let result = image_to_video(request)
.with_api_key(&args.api_key.0)
.queue()
.await?;
.await;
let result = match result {
Ok(result) => result,
Err(err) => return Err(classify_fal_error(err)),
};
Ok(EnqueuedRequest::from_queue_response(&result)?)
}
@@ -57,7 +57,7 @@ pub async fn enqueue_kling_16_image_to_video<P: AsRef<Path>>(args: Kling16Args<'
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use crate::requests::video_gen::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration};
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
@@ -74,6 +74,7 @@ mod tests {
let args = Kling16Args {
image_path: image,
prompt: "a corgi looks out over the water",
api_key: &api_key,
duration: Kling16Duration::Default,
};
@@ -0,0 +1,71 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use fal::endpoints::fal_ai::minimax::image_to_video::image_to_video;
use fal::endpoints::fal_ai::minimax::image_to_video::VideoOutput;
use fal::endpoints::fal_ai::minimax::image_to_video::ImageToVideoRequest;
use fal_client::file_to_base64_url::file_to_base64_url;
use futures::StreamExt;
use std::io::Write;
use std::path::Path;
pub struct Minimax01Args<'a, P: AsRef<Path>> {
pub image_path: P,
pub prompt: &'a str,
pub api_key: &'a FalApiKey,
}
pub async fn enqueue_minimax_01_image_to_video<P: AsRef<Path>>(args: Minimax01Args<'_, P>) -> Result<EnqueuedRequest, FalErrorPlus> {
let image_url = file_to_base64_url(args.image_path)?;
let request = ImageToVideoRequest {
image_url,
prompt: args.prompt.to_string(),
prompt_optimizer: None,
};
let result = image_to_video(request)
.with_api_key(&args.api_key.0)
.queue()
.await;
let result = match result {
Ok(result) => result,
Err(err) => return Err(classify_fal_error(err)),
};
Ok(EnqueuedRequest::from_queue_response(&result)?)
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use errors::AnyhowResult;
use std::fs::read_to_string;
use testing::test_file_path::test_file_path;
use crate::requests::video_gen::enqueue_minimax_01_image_to_video::{enqueue_minimax_01_image_to_video, Minimax01Args};
#[tokio::test]
#[ignore]
async fn test_minimax_video() -> AnyhowResult<()> {
let image = test_file_path("test_data/image/juno.jpg")?;
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = Minimax01Args {
image_path: image,
prompt: "a dog jumps into the water",
api_key: &api_key,
};
let result = enqueue_minimax_01_image_to_video(args).await?;
println!("Request ID: {:?}", result.request_id);
Ok(())
}
}
@@ -0,0 +1,2 @@
pub mod enqueue_kling_16_image_to_video;
pub mod enqueue_minimax_01_image_to_video;
@@ -1,9 +1,13 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::fal_error_plus::FalErrorPlus;
use crate::error::fal_error_plus::FalErrorPlus;
use crate::model::enqueued_request::EnqueuedRequest;
use crate::model::fal_endpoint::FalEndpoint;
use anyhow::anyhow;
use fal::endpoints::fal_ai::flux_pro::v1_1_ultra::Output;
use fal::endpoints::fal_ai::hunyuan3d::v2::ObjectOutput;
use fal::endpoints::fal_ai::kling_video::v1_6::pro::effects::I2VOutput;
use fal::endpoints::fal_ai::minimax::image_to_video::VideoOutput;
use fal::endpoints::fal_ai::recraft_v3::TextToImageOutput;
use fal::prelude::QueueResponse;
use fal::queue::{Queue, QueueStatus};
use reqwest::Client;
@@ -20,6 +24,11 @@ impl QueueStatusChecker {
pub async fn check_status(&self, request: &EnqueuedRequest) -> Result<QueueStatus, FalErrorPlus> {
match &request.fal_endpoint {
FalEndpoint::FluxProUltraTextToImage => {
let queue = self.make_queue::<Output>(&request);
let status = queue.status(false).await?;
Ok(status)
}
FalEndpoint::Hunyuan3d2Base => {
let queue = self.make_queue::<ObjectOutput>(&request);
let status = queue.status(false).await?;
@@ -30,6 +39,16 @@ impl QueueStatusChecker {
let status = queue.status(false).await?;
Ok(status)
}
FalEndpoint::Minimax01ImageToVideo => {
let queue = self.make_queue::<VideoOutput>(&request); // Assuming Minimax uses the same output type
let status = queue.status(false).await?;
Ok(status)
}
FalEndpoint::RecraftV3TextToImage => {
let queue = self.make_queue::<TextToImageOutput>(&request);
let status = queue.status(false).await?;
Ok(status)
}
FalEndpoint::Other(endpoint) => {
Err(FalErrorPlus::UnhandledEndpoint(format!("Unsupported endpoint: {}", endpoint)))
}
@@ -38,15 +57,35 @@ impl QueueStatusChecker {
pub async fn get_download_url(&self, request: &EnqueuedRequest) -> Result<String, FalErrorPlus> {
match &request.fal_endpoint {
FalEndpoint::FluxProUltraTextToImage => {
let queue = self.make_queue::<Output>(&request);
let response = queue.response().await?;
// TODO(bt,2025-05-26): Handle multiple images
Ok(response.images.first()
.ok_or_else(|| FalErrorPlus::AnyhowError(anyhow!("No images returned")))?
.url.clone())
}
FalEndpoint::Hunyuan3d2Base => {
let queue = self.make_queue::<ObjectOutput>(&request);
let status = queue.response().await?;
Ok(status.model_mesh.url)
let response = queue.response().await?;
Ok(response.model_mesh.url)
}
FalEndpoint::Kling16ImageToVideo => {
let queue = self.make_queue::<I2VOutput>(&request);
let status = queue.response().await?;
Ok(status.video.url)
let response = queue.response().await?;
Ok(response.video.url)
}
FalEndpoint::Minimax01ImageToVideo => {
let queue = self.make_queue::<VideoOutput>(&request); // Assuming Minimax uses the same output type
let response = queue.response().await?;
Ok(response.video.url)
}
FalEndpoint::RecraftV3TextToImage => {
let queue = self.make_queue::<TextToImageOutput>(&request);
let response = queue.response().await?;
Ok(response.images.first()
.ok_or_else(|| FalErrorPlus::AnyhowError(anyhow!("No images returned")))?
.url.clone())
}
FalEndpoint::Other(endpoint) => {
Err(FalErrorPlus::UnhandledEndpoint(format!("Unsupported endpoint: {}", endpoint)))
@@ -13,7 +13,7 @@ interface TextToImageProps {
const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
const containerRef = useRef<HTMLDivElement>(null);
const [selectedModel, setSelectedModel] = useState<string>("Kling 2.0");
const [selectedModel, setSelectedModel] = useState<string>("Flux Pro Ultra");
const [useSystemPrompt, setUseSystemPrompt] = useState(false);
const [isImageBeingProcessed, setIsImageBeingProcessed] = useState(false);
const [selectedImage, setSelectedImage] = useState<File | null>(null);
@@ -29,16 +29,16 @@ const TextToImage = ({ imageMediaId, imageUrl }: TextToImageProps) => {
const modelList: PopoverItem[] = [
{
label: "Kling 2.0",
label: "Flux Pro Ultra",
icon: <FontAwesomeIcon icon={faFilm} className="h-4 w-4" />,
selected: selectedModel === "Kling 2.0",
selected: selectedModel === "Flux Pro Ultra",
description: "High quality model",
badges: [{ label: "2 min.", icon: <FontAwesomeIcon icon={faClock} /> }],
},
{
label: "Runway",
label: "Recraft 3",
icon: <FontAwesomeIcon icon={faFilm} className="h-4 w-4" />,
selected: selectedModel === "Runway",
selected: selectedModel === "Recraft 3",
description: "Fast and high-quality model",
badges: [{ label: "2 min.", icon: <FontAwesomeIcon icon={faClock} /> }],
},
@@ -7,7 +7,7 @@ import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover";
import { Tooltip } from "@storyteller/ui-tooltip";
import { Button, ToggleButton } from "@storyteller/ui-button";
import { Modal } from "@storyteller/ui-modal";
import { FalKlingImageToVideo } from "@storyteller/tauri-api";
import { EnqueueTextToImage, EnqueueTextToImageModel } from "@storyteller/tauri-api";
import {
faMessageXmark,
faMessageCheck,
@@ -71,8 +71,8 @@ export const PromptBoxImage = ({
console.log("Is this a desktop app?", IsDesktopApp());
const [isModalOpen, setIsModalOpen] = useState(false);
const [content, setContent] = useState("");
const { jobTokens, addJobToken, removeJobToken, clearJobTokens } =
useJobContext();
//const { jobTokens, addJobToken, removeJobToken, clearJobTokens } =
// useJobContext();
const [prompt, setPrompt] = useState("");
const [isEnqueueing, setIsEnqueueing] = useState(false);
@@ -199,12 +199,15 @@ export const PromptBoxImage = ({
const handleEnqueue = async () => {
setIsEnqueueing(true);
console.log("generateResponse", generateResponse);
const modelName = getModelByName(model);
const generateResponse = await FalKlingImageToVideo({
image_media_token: referenceImages[0].mediaToken,
const generateResponse = await EnqueueTextToImage({
prompt: prompt,
model: modelName,
});
console.log("generateResponse", generateResponse);
onEnqueuePressed?.();
if (generateResponse.status === CommandSuccessStatus.Success) {
const prefs = await GetAppPreferences();
@@ -259,6 +262,17 @@ export const PromptBoxImage = ({
return iconElement.props.icon;
};
const getModelByName = (name: string) : EnqueueTextToImageModel => {
switch (name) {
case "Flux Pro Ultra":
return EnqueueTextToImageModel.FluxProUltra;
case "Recraft 3":
return EnqueueTextToImageModel.Recraft3;
default:
return EnqueueTextToImageModel.FluxProUltra;
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
+1
View File
@@ -1,4 +1,5 @@
export * from "./lib/common/CommandStatus.js";
export * from "./lib/enqueue/EnqueueTextToImage.js";
export * from "./lib/fal/FalBackgroundRemoval.js";
export * from "./lib/fal/FalHunyuanImageTo3d.js";
export * from "./lib/fal/FalKlingImageToVideo.js";