From 4f0916edb769db122daa4f8fb6b4983bd5959a40 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 29 Jul 2025 20:14:10 -0400 Subject: [PATCH] enqueue multiple images (#749) --- ...queue_gpt_image_1_text_to_image_webhook.rs | 123 ++++++++++ .../src/requests/webhook/image/mod.rs | 1 + .../generate_gpt_image_1_text_to_image.rs | 19 ++ .../src/generate/image/mod.rs | 1 + .../image/enqueue_text_to_image_command.rs | 17 +- .../image/generic/handle_image_artcraft.rs | 114 ++++++---- .../image/gpt_image_1/handle_gpt_image_1.rs | 75 ++++++ .../handle_gpt_image_1_artcraft.rs | 115 ++++++++++ ...age_sora.rs => handle_gpt_image_1_sora.rs} | 15 +- .../commands/enqueue/image/gpt_image_1/mod.rs | 4 +- .../enqueue_image_bg_removal_command.rs | 2 +- .../generic/handle_generic_bg_removal.rs | 4 +- .../enqueue_contextual_edit_image_command.rs | 4 +- ..._image_1.rs => handle_gpt_image_1_edit.rs} | 14 +- ...rs => handle_gpt_image_1_edit_artcraft.rs} | 6 +- ...ora.rs => handle_gpt_image_1_edit_sora.rs} | 4 +- .../enqueue/image_edit/gpt_image_1/mod.rs | 6 +- crates/lib/files/images/Cargo.toml | 2 +- crates/lib/files/images/src/encoding/mod.rs | 1 + .../src/encoding/webp_bytes_to_png_bytes.rs | 21 ++ crates/lib/files/images/src/error.rs | 31 +++ crates/lib/files/images/src/lib.rs | 2 + .../generate_gpt_image_1_text_to_image.rs | 58 +++++ .../src/generate/image/mod.rs | 1 + .../web/storyteller_web/src/docs/api_doc.rs | 11 + ...erate_gpt_image_1_text_to_image_handler.rs | 213 ++++++++++++++++++ .../endpoints/generate/image/mod.rs | 1 + .../webhooks/handle_image_payload.rs | 54 ++++- .../webhooks/handle_images_payload.rs | 53 ++++- .../http_server/routes/add_generate_routes.rs | 7 +- .../fal_ai/gpt_image_1/edit_image/byok/mod.rs | 20 ++ .../src/lib/enqueue/EnqueueTextToImage.ts | 6 + 32 files changed, 925 insertions(+), 80 deletions(-) create mode 100644 crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_text_to_image_webhook.rs create mode 100644 crates/api_clients/storyteller_client/src/generate/image/generate_gpt_image_1_text_to_image.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_artcraft.rs rename crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/{handle_image_sora.rs => handle_gpt_image_1_sora.rs} (92%) rename crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/{handle_gpt_image_1.rs => handle_gpt_image_1_edit.rs} (87%) rename crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/{handle_gpt_image_1_artcraft.rs => handle_gpt_image_1_edit_artcraft.rs} (98%) rename crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/{handle_gpt_image_1_sora.rs => handle_gpt_image_1_edit_sora.rs} (99%) create mode 100644 crates/lib/files/images/src/encoding/mod.rs create mode 100644 crates/lib/files/images/src/encoding/webp_bytes_to_png_bytes.rs create mode 100644 crates/lib/files/images/src/error.rs create mode 100644 crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_gpt_image_1_text_to_image.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_gpt_image_1_text_to_image_handler.rs diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_text_to_image_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_text_to_image_webhook.rs new file mode 100644 index 0000000000..f816e679af --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_text_to_image_webhook.rs @@ -0,0 +1,123 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::creds::open_ai_api_key::OpenAiApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::prelude::fal_ai::gpt_image_1::edit_image::byok::{gpt_text_to_image, GptTextToImageRequest}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct GptTextToImageByokArgs<'a, V: IntoUrl> { + // Request + pub prompt: &'a str, + pub image_size: GptTextToImageSize, + pub num_images: GptTextToImageNumImages, + pub quality: GptTextToImageQuality, + + // Fulfillment + pub api_key: &'a FalApiKey, + pub openai_api_key: &'a OpenAiApiKey, + pub webhook_url: V, +} + +#[derive(Copy, Clone, Debug)] +pub enum GptTextToImageSize{ + Auto, + Square, + Horizontal, + Vertical, +} + +#[derive(Copy, Clone, Debug)] +pub enum GptTextToImageQuality { + Auto, + Low, + Medium, + High, +} + +#[derive(Copy, Clone, Debug)] +pub enum GptTextToImageNumImages{ + One, + Two, + Three, + Four, +} + +pub async fn enqueue_gpt_image_1_text_to_image_webhook( + args: GptTextToImageByokArgs<'_, V> +) -> Result { + + // auto, 1024x1024, 1536x1024, 1024x1536 + let image_size = match args.image_size { + GptTextToImageSize::Auto => "auto", + GptTextToImageSize::Square => "1024x1024", + GptTextToImageSize::Horizontal => "1536x1024", + GptTextToImageSize::Vertical => "1024x1536", + }; + + let quality = match args.quality { + GptTextToImageQuality::Auto => "auto", + GptTextToImageQuality::Low => "low", + GptTextToImageQuality::Medium => "medium", + GptTextToImageQuality::High => "high", + }; + + let num_images = match args.num_images { + GptTextToImageNumImages::One => 1, + GptTextToImageNumImages::Two => 2, + GptTextToImageNumImages::Three => 3, + GptTextToImageNumImages::Four => 4, + }; + + let request = GptTextToImageRequest { + prompt: args.prompt.to_string(), + image_size: image_size.to_string(), + num_images, + quality: quality.to_string(), + openai_api_key: args.openai_api_key.0.to_string(), + }; + + let result = gpt_text_to_image(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::creds::open_ai_api_key::OpenAiApiKey; + use crate::requests::webhook::image::enqueue_gpt_image_1_text_to_image_webhook::{enqueue_gpt_image_1_text_to_image_webhook, GptTextToImageByokArgs, GptTextToImageNumImages, GptTextToImageQuality, GptTextToImageSize}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test_gpt_image_1_edit_image() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let fal_api_key = FalApiKey::from_str(&secret); + + let secret = read_to_string("/Users/bt/Artcraft/credentials/openai_api_key.txt")?; + + let open_ai_api_key = OpenAiApiKey::from_str(&secret); + + let args = GptTextToImageByokArgs { + prompt: "put the man and the ghost on the grassy hill. the man is scared of the friendly ghost.", + api_key: &fal_api_key, + openai_api_key: &open_ai_api_key, + webhook_url: "https://example.com/webhook", + image_size: GptTextToImageSize::Horizontal, + num_images: GptTextToImageNumImages::One, + quality: GptTextToImageQuality::High, + }; + + let result = enqueue_gpt_image_1_text_to_image_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs index 33a9db88db..f2f10beab0 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs @@ -3,4 +3,5 @@ pub mod enqueue_flux_1_schnell_text_to_image_webhook; pub mod enqueue_flux_pro_11_text_to_image_webhook; pub mod enqueue_flux_pro_11_ultra_text_to_image_webhook; pub mod enqueue_gpt_image_1_edit_image_webhook; +pub mod enqueue_gpt_image_1_text_to_image_webhook; pub mod remove_background_rembg_webhook; diff --git a/crates/api_clients/storyteller_client/src/generate/image/generate_gpt_image_1_text_to_image.rs b/crates/api_clients/storyteller_client/src/generate/image/generate_gpt_image_1_text_to_image.rs new file mode 100644 index 0000000000..2edacc3435 --- /dev/null +++ b/crates/api_clients/storyteller_client/src/generate/image/generate_gpt_image_1_text_to_image.rs @@ -0,0 +1,19 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageRequest, GenerateGptImage1TextToImageResponse, GPT_IMAGE_1_TEXT_TO_IMAGE_PATH}; + + +pub async fn generate_gpt_image_1_text_to_image( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: GenerateGptImage1TextToImageRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + GPT_IMAGE_1_TEXT_TO_IMAGE_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/storyteller_client/src/generate/image/mod.rs b/crates/api_clients/storyteller_client/src/generate/image/mod.rs index 1d9185f67c..90f2d43e51 100644 --- a/crates/api_clients/storyteller_client/src/generate/image/mod.rs +++ b/crates/api_clients/storyteller_client/src/generate/image/mod.rs @@ -3,4 +3,5 @@ pub mod generate_flux_1_dev_text_to_image; pub mod generate_flux_1_schnell_text_to_image; pub mod generate_flux_pro_11_text_to_image; pub mod generate_flux_pro_11_ultra_text_to_image; +pub mod generate_gpt_image_1_text_to_image; pub mod remove_image_background; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs index 8398efcc75..b017fa5816 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs @@ -1,6 +1,7 @@ use crate::core::commands::enqueue::image::generic::handle_image_artcraft::handle_image_artcraft; use crate::core::commands::enqueue::image::generic::handle_image_fal::handle_image_fal; -use crate::core::commands::enqueue::image::gpt_image_1::handle_image_sora::handle_image_sora; +use crate::core::commands::enqueue::image::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1; +use crate::core::commands::enqueue::image::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora; use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; @@ -39,6 +40,9 @@ pub struct EnqueueTextToImageRequest { /// Aspect ratio. pub aspect_ratio: Option, + + /// The number of images to generate. + pub number_images: Option, } // TODO(bt,2025-07-14): Support other aspect ratios / resolutions - @@ -224,7 +228,16 @@ pub async fn dispatch_request( return Err(InternalImageError::NoModelSpecified); } Some(ImageModel::GptImage1) => { - return handle_image_sora(&app, request, sora_creds_manager, sora_task_queue).await; + return handle_gpt_image_1( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + sora_creds_manager, + sora_task_queue, + ).await; } _ => { // Fall-through diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/generic/handle_image_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/generic/handle_image_artcraft.rs index d9f32acecc..2423fafc45 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/generic/handle_image_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/generic/handle_image_artcraft.rs @@ -9,10 +9,10 @@ use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use anyhow::anyhow; -use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::{GenerateFlux1DevTextToImageAspectRatio, GenerateFlux1DevTextToImageRequest}; -use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::{GenerateFlux1SchnellTextToImageAspectRatio, GenerateFlux1SchnellTextToImageRequest}; -use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::{GenerateFluxPro11TextToImageAspectRatio, GenerateFluxPro11TextToImageRequest}; -use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::{GenerateFluxPro11UltraTextToImageAspectRatio, GenerateFluxPro11UltraTextToImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::{GenerateFlux1DevTextToImageAspectRatio, GenerateFlux1DevTextToImageNumImages, GenerateFlux1DevTextToImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::{GenerateFlux1SchnellTextToImageAspectRatio, GenerateFlux1SchnellTextToImageNumImages, GenerateFlux1SchnellTextToImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::{GenerateFluxPro11TextToImageAspectRatio, GenerateFluxPro11TextToImageNumImages, GenerateFluxPro11TextToImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::{GenerateFluxPro11UltraTextToImageAspectRatio, GenerateFluxPro11UltraTextToImageNumImages, GenerateFluxPro11UltraTextToImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; @@ -58,7 +58,7 @@ pub async fn handle_image_artcraft( return Err(InternalImageError::NoModelSpecified); } Some(ImageModel::GptImage1) => { - return Err(InternalImageError::AnyhowError(anyhow!("wrong logic: artcraft is handling sora images"))); + return Err(InternalImageError::AnyhowError(anyhow!("wrong logic: an earlier branch should have handled gpt-image-1 images for Artcraft"))); } Some(ImageModel::Recraft3) => { return Err(InternalImageError::AnyhowError(anyhow!("not yet implemented in Artcraft"))); @@ -66,19 +66,25 @@ pub async fn handle_image_artcraft( Some(ImageModel::Flux1Dev) => { info!("enqueue Flux 1 Dev"); selected_model = GenerationModel::Flux1Dev; - let aspect_ratio = request.aspect_ratio - .map(|aspect| match aspect { - // TODO(bt,2025-07-14): Support other aspect ratios. - TextToImageSize::Tall => GenerateFlux1DevTextToImageAspectRatio::PortraitNineBySixteen, - TextToImageSize::Wide => GenerateFlux1DevTextToImageAspectRatio::LandscapeSixteenByNine, - TextToImageSize::Auto => GenerateFlux1DevTextToImageAspectRatio::SquareHd, - TextToImageSize::Square => GenerateFlux1DevTextToImageAspectRatio::SquareHd, - }); let request = GenerateFlux1DevTextToImageRequest { uuid_idempotency_token, prompt: request.prompt, - aspect_ratio, - num_images: None, + aspect_ratio: request.aspect_ratio + .map(|aspect| match aspect { + // TODO(bt,2025-07-14): Support other aspect ratios. + TextToImageSize::Tall => GenerateFlux1DevTextToImageAspectRatio::PortraitNineBySixteen, + TextToImageSize::Wide => GenerateFlux1DevTextToImageAspectRatio::LandscapeSixteenByNine, + TextToImageSize::Auto => GenerateFlux1DevTextToImageAspectRatio::SquareHd, + TextToImageSize::Square => GenerateFlux1DevTextToImageAspectRatio::SquareHd, + }), + num_images: request.number_images + .and_then(|num| match num { + 1 => Some(GenerateFlux1DevTextToImageNumImages::One), + 2 => Some(GenerateFlux1DevTextToImageNumImages::Two), + 3 => Some(GenerateFlux1DevTextToImageNumImages::Three), + 4 => Some(GenerateFlux1DevTextToImageNumImages::Four), + _ => None, // NB: use service defaults + }), }; let result = generate_flux_1_dev_text_to_image( &app_env_configs.storyteller_host, @@ -99,19 +105,25 @@ pub async fn handle_image_artcraft( Some(ImageModel::Flux1Schnell) => { info!("enqueue Flux 1 Schnell"); selected_model = GenerationModel::Flux1Schnell; - let aspect_ratio = request.aspect_ratio - .map(|aspect| match aspect { - // TODO(bt,2025-07-14): Support other aspect ratios. - TextToImageSize::Tall => GenerateFlux1SchnellTextToImageAspectRatio::PortraitNineBySixteen, - TextToImageSize::Wide => GenerateFlux1SchnellTextToImageAspectRatio::LandscapeSixteenByNine, - TextToImageSize::Auto => GenerateFlux1SchnellTextToImageAspectRatio::SquareHd, - TextToImageSize::Square => GenerateFlux1SchnellTextToImageAspectRatio::SquareHd, - }); let request = GenerateFlux1SchnellTextToImageRequest { uuid_idempotency_token, prompt: request.prompt, - aspect_ratio, - num_images: None, + aspect_ratio: request.aspect_ratio + .map(|aspect| match aspect { + // TODO(bt,2025-07-14): Support other aspect ratios. + TextToImageSize::Tall => GenerateFlux1SchnellTextToImageAspectRatio::PortraitNineBySixteen, + TextToImageSize::Wide => GenerateFlux1SchnellTextToImageAspectRatio::LandscapeSixteenByNine, + TextToImageSize::Square => GenerateFlux1SchnellTextToImageAspectRatio::SquareHd, + TextToImageSize::Auto => GenerateFlux1SchnellTextToImageAspectRatio::SquareHd, + }), + num_images: request.number_images + .and_then(|num| match num { + 1 => Some(GenerateFlux1SchnellTextToImageNumImages::One), + 2 => Some(GenerateFlux1SchnellTextToImageNumImages::Two), + 3 => Some(GenerateFlux1SchnellTextToImageNumImages::Three), + 4 => Some(GenerateFlux1SchnellTextToImageNumImages::Four), + _ => None, // NB: use service defaults + }), }; let result = generate_flux_1_schnell_text_to_image( &app_env_configs.storyteller_host, @@ -132,19 +144,25 @@ pub async fn handle_image_artcraft( Some(ImageModel::FluxPro11) => { info!("enqueue Flux Pro 1.1"); selected_model = GenerationModel::FluxPro11; - let aspect_ratio = request.aspect_ratio - .map(|aspect| match aspect { - // TODO(bt,2025-07-14): Support other aspect ratios. - TextToImageSize::Tall => GenerateFluxPro11TextToImageAspectRatio::PortraitNineBySixteen, - TextToImageSize::Wide => GenerateFluxPro11TextToImageAspectRatio::LandscapeSixteenByNine, - TextToImageSize::Auto => GenerateFluxPro11TextToImageAspectRatio::Square, - TextToImageSize::Square => GenerateFluxPro11TextToImageAspectRatio::Square, - }); let request = GenerateFluxPro11TextToImageRequest { uuid_idempotency_token, prompt: request.prompt, - aspect_ratio, - num_images: None, + aspect_ratio: request.aspect_ratio + .map(|aspect| match aspect { + // TODO(bt,2025-07-14): Support other aspect ratios. + TextToImageSize::Tall => GenerateFluxPro11TextToImageAspectRatio::PortraitNineBySixteen, + TextToImageSize::Wide => GenerateFluxPro11TextToImageAspectRatio::LandscapeSixteenByNine, + TextToImageSize::Square => GenerateFluxPro11TextToImageAspectRatio::Square, + TextToImageSize::Auto => GenerateFluxPro11TextToImageAspectRatio::Square, + }), + num_images: request.number_images + .and_then(|num| match num { + 1 => Some(GenerateFluxPro11TextToImageNumImages::One), + 2 => Some(GenerateFluxPro11TextToImageNumImages::Two), + 3 => Some(GenerateFluxPro11TextToImageNumImages::Three), + 4 => Some(GenerateFluxPro11TextToImageNumImages::Four), + _ => None, // NB: use service defaults + }), }; let result = generate_flux_pro_11_text_to_image( &app_env_configs.storyteller_host, @@ -165,19 +183,25 @@ pub async fn handle_image_artcraft( Some(ImageModel::FluxPro11Ultra) => { info!("enqueue Flux Pro 1.1 Ultra"); selected_model = GenerationModel::FluxPro11Ultra; - let aspect_ratio = request.aspect_ratio - .map(|aspect| match aspect { - // TODO(bt,2025-07-14): Support other aspect ratios. - TextToImageSize::Tall => GenerateFluxPro11UltraTextToImageAspectRatio::PortraitNineBySixteen, - TextToImageSize::Wide => GenerateFluxPro11UltraTextToImageAspectRatio::LandscapeSixteenByNine, - TextToImageSize::Auto => GenerateFluxPro11UltraTextToImageAspectRatio::Square, - TextToImageSize::Square => GenerateFluxPro11UltraTextToImageAspectRatio::Square, - }); let request = GenerateFluxPro11UltraTextToImageRequest { uuid_idempotency_token, prompt: request.prompt, - aspect_ratio, - num_images: None, + aspect_ratio: request.aspect_ratio + .map(|aspect| match aspect { + // TODO(bt,2025-07-14): Support other aspect ratios. + TextToImageSize::Tall => GenerateFluxPro11UltraTextToImageAspectRatio::PortraitNineBySixteen, + TextToImageSize::Wide => GenerateFluxPro11UltraTextToImageAspectRatio::LandscapeSixteenByNine, + TextToImageSize::Square => GenerateFluxPro11UltraTextToImageAspectRatio::Square, + TextToImageSize::Auto => GenerateFluxPro11UltraTextToImageAspectRatio::Square, + }), + num_images: request.number_images + .and_then(|num| match num { + 1 => Some(GenerateFluxPro11UltraTextToImageNumImages::One), + 2 => Some(GenerateFluxPro11UltraTextToImageNumImages::Two), + 3 => Some(GenerateFluxPro11UltraTextToImageNumImages::Three), + 4 => Some(GenerateFluxPro11UltraTextToImageNumImages::Four), + _ => None, // NB: use service defaults + }), }; let result = generate_flux_pro_11_ultra_text_to_image( &app_env_configs.storyteller_host, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1.rs new file mode 100644 index 0000000000..c4ce6cb318 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1.rs @@ -0,0 +1,75 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; +use crate::core::commands::enqueue::image::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft; +use crate::core::commands::enqueue::image::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora; +use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::model::image_models::ImageModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use log::{error, info}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use std::time::Duration; +use tauri::AppHandle; + +pub async fn handle_gpt_image_1( + request: EnqueueTextToImageRequest, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough + // If in the future we support OpenAI API keys, it's worth considering whether we + // send those requests ourselves, or if we use FAL as an intermediary. FAL makes + // the API nicer to deal with, but the user needs an additional key. + } + Provider::Artcraft => { + info!("Dispatching gpt-image-1 via Artcraft..."); + return handle_gpt_image_1_artcraft( + request, + app_env_configs, + storyteller_creds_manager + ).await; + } + Provider::Sora => { + info!("Dispatching gpt-image-1 via Sora..."); + return handle_gpt_image_1_sora( + request, + app, + sora_creds_manager, + sora_task_queue, + ).await; + } + } + } + + Err(InternalImageError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_artcraft.rs new file mode 100644 index 0000000000..3d864871cc --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_artcraft.rs @@ -0,0 +1,115 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; +use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::model::image_models::ImageModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageImageQuality, GenerateGptImage1TextToImageImageSize, GenerateGptImage1TextToImageNumImages, GenerateGptImage1TextToImageRequest}; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use idempotency::uuid::generate_random_uuid; +use log::{error, info}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use std::time::Duration; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::generate::image::generate_gpt_image_1_text_to_image::generate_gpt_image_1_text_to_image; +use tauri::AppHandle; + +pub async fn handle_gpt_image_1_artcraft( + request: EnqueueTextToImageRequest, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalImageError::NeedsStorytellerCredentials); + }, + }; + + info!("Calling Artcraft gpt-image-1 ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let image_quality = Some(GenerateGptImage1TextToImageImageQuality::High); + + //let image_quality = request.image_quality + // .map(|quality| match quality { + // EditImageQuality::Auto => GenerateGptImage1TextToImageImageQuality::Auto, + // EditImageQuality::High => GenerateGptImage1TextToImageImageQuality::High, + // EditImageQuality::Medium => GenerateGptImage1TextToImageImageQuality::Medium, + // EditImageQuality::Low => GenerateGptImage1TextToImageImageQuality::Low, + // }); + + let image_size = request.aspect_ratio + .map(|size| match size { + TextToImageSize::Auto => GenerateGptImage1TextToImageImageSize::Square, + TextToImageSize::Square => GenerateGptImage1TextToImageImageSize::Square, + TextToImageSize::Tall => GenerateGptImage1TextToImageImageSize::Vertical, + TextToImageSize::Wide => GenerateGptImage1TextToImageImageSize::Horizontal, + }); + + let num_images = match request.number_images { + None => None, + Some(1) => Some(GenerateGptImage1TextToImageNumImages::One), + Some(2) => Some(GenerateGptImage1TextToImageNumImages::Two), + Some(3) => Some(GenerateGptImage1TextToImageNumImages::Three), + Some(4) => Some(GenerateGptImage1TextToImageNumImages::Four), + // TODO: Error + //Some(other) => { + // return Err(InternalImageError::InvalidNumberOfRequestedImages { + // min: 1, + // max: 4, + // requested: other, + // }); + //}, + _ => Some(GenerateGptImage1TextToImageNumImages::One), // Default to one image if invalid number + }; + + let request = GenerateGptImage1TextToImageRequest { + uuid_idempotency_token, + prompt: request.prompt.clone(), + image_size, + num_images, + image_quality, + }; + + let result = generate_gpt_image_1_text_to_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + let job_id = match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft gpt-image-1. Job token: {}", + enqueued.inference_job_token); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft gpt-image-1: {:?}", err); + return Err(InternalImageError::StorytellerError(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Artcraft, + model: Some(GenerationModel::GptImage1), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::ImageGeneration, + }) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_image_sora.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_sora.rs similarity index 92% rename from crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_image_sora.rs rename to crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_sora.rs index 34a206843b..6bc35a4d1b 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_image_sora.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/handle_gpt_image_1_sora.rs @@ -19,9 +19,9 @@ use tauri::AppHandle; const SORA_SIMPLE_IMAGE_GEN_TIMEOUT : Duration = Duration::from_millis(1000 * 30); // 30 seconds -pub async fn handle_image_sora( - app: &AppHandle, +pub async fn handle_gpt_image_1_sora( request: EnqueueTextToImageRequest, + app: &AppHandle, sora_creds_manager: &SoraCredentialManager, sora_task_queue: &SoraTaskQueue, ) -> Result { @@ -55,10 +55,19 @@ pub async fn handle_image_sora( Some(TextToImageSize::Wide) => ImageSize::Wide, }; + let num_images = match request.number_images { + None => NumImages::One, + Some(1) => NumImages::One, + Some(2) => NumImages::Two, + Some(3) => NumImages::Two, // NB: There is no option for three images! + Some(4) => NumImages::Four, + _ => NumImages::One, + }; + let result = simple_image_gen_with_session_auto_renew(SimpleImageGenAutoRenewRequest { prompt: prompt.to_string(), - num_images: NumImages::One, + num_images, image_size: size, credentials: &creds, request_timeout: Some(SORA_SIMPLE_IMAGE_GEN_TIMEOUT), diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/mod.rs index b07116581b..692ac03133 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/gpt_image_1/mod.rs @@ -1 +1,3 @@ -pub mod handle_image_sora; +pub (crate) mod handle_gpt_image_1_artcraft; +pub (crate) mod handle_gpt_image_1_sora; +pub (super) mod handle_gpt_image_1; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/enqueue_image_bg_removal_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/enqueue_image_bg_removal_command.rs index 9c53b4b806..8e1d630e79 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/enqueue_image_bg_removal_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/enqueue_image_bg_removal_command.rs @@ -3,7 +3,7 @@ use crate::core::commands::enqueue::image::internal_image_error::InternalImageEr use crate::core::commands::enqueue::image_bg_removal::errors::InternalBgRemovalError; use crate::core::commands::enqueue::image_bg_removal::generic::handle_generic_bg_removal::handle_generic_bg_removal; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::handle_gpt_image_1_edit; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType}; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal.rs index 75eeacc3db..620b19b451 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal.rs @@ -4,8 +4,8 @@ use crate::core::commands::enqueue::image_bg_removal::generic::handle_generic_bg use crate::core::commands::enqueue::image_bg_removal::generic::handle_generic_bg_removal_fal::handle_generic_bg_removal_fal; use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; use crate::core::state::data_dir::app_data_root::AppDataRoot; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs index 75a4b00acb..c579f1c7e7 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs @@ -1,7 +1,7 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, EnqueueTextToImageSuccessResponse}; use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::handle_gpt_image_1_edit; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType}; @@ -208,7 +208,7 @@ pub async fn handle_request( return Err(InternalContextualEditImageError::NoModelSpecified) } Some(ContextualImageEditModel::GptImage1) => { - handle_gpt_image_1( + handle_gpt_image_1_edit( request, app, app_data_root, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit.rs similarity index 87% rename from crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs rename to crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit.rs index cb63e2fc3a..45a7f0a344 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit.rs @@ -1,7 +1,7 @@ use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; use crate::core::state::data_dir::app_data_root::AppDataRoot; @@ -16,7 +16,7 @@ use tauri::AppHandle; pub(super) const MAX_IMAGES: usize = 10; -pub async fn handle_gpt_image_1( +pub async fn handle_gpt_image_1_edit( request: &EnqueueContextualEditImageCommand, app: &AppHandle, app_data_root: &AppDataRoot, @@ -44,8 +44,8 @@ pub async fn handle_gpt_image_1( // the API nicer to deal with, but the user needs an additional key. } Provider::Artcraft => { - info!("Dispatching gpt-image-1 via Artcraft..."); - return handle_gpt_image_1_artcraft( + info!("Dispatching gpt-image-1 (edit) via Artcraft..."); + return handle_gpt_image_1_edit_artcraft( request, app, app_data_root, @@ -54,8 +54,8 @@ pub async fn handle_gpt_image_1( ).await; } Provider::Sora => { - info!("Dispatching gpt-image-1 via Sora..."); - return handle_gpt_image_1_sora( + info!("Dispatching gpt-image-1 (edit) via Sora..."); + return handle_gpt_image_1_edit_sora( request, app, app_data_root, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs similarity index 98% rename from crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs rename to crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs index 6250f2be66..1559a38131 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs @@ -2,7 +2,7 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::Enqueu use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::MAX_IMAGES; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::MAX_IMAGES; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; @@ -36,7 +36,7 @@ use tauri::AppHandle; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -pub async fn handle_gpt_image_1_artcraft( +pub async fn handle_gpt_image_1_edit_artcraft( request: &EnqueueContextualEditImageCommand, app: &AppHandle, app_data_root: &AppDataRoot, @@ -51,7 +51,7 @@ pub async fn handle_gpt_image_1_artcraft( }, }; - info!("Calling Artcraft gpt-image-1 ..."); + info!("Calling Artcraft gpt-image-1 (edit) ..."); let uuid_idempotency_token = generate_random_uuid(); diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_sora.rs similarity index 99% rename from crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs rename to crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_sora.rs index ba3c6af003..6d3a4d3166 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_sora.rs @@ -2,7 +2,7 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::Enqueu use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageSize, EnqueueContextualEditImageCommand, EnqueueContextualEditImageErrorType}; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; -use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::MAX_IMAGES; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::MAX_IMAGES; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; @@ -48,7 +48,7 @@ const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // const DEFAULT_ASPECT_RATIO : EditImageSize = EditImageSize::Square; -pub async fn handle_gpt_image_1_sora( +pub async fn handle_gpt_image_1_edit_sora( request: &EnqueueContextualEditImageCommand, app: &AppHandle, app_data_root: &AppDataRoot, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs index 73b5e0a985..0aaff118a9 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs @@ -1,3 +1,3 @@ -pub mod handle_gpt_image_1; -pub mod handle_gpt_image_1_artcraft; -pub mod handle_gpt_image_1_sora; \ No newline at end of file +pub mod handle_gpt_image_1_edit; +pub mod handle_gpt_image_1_edit_artcraft; +pub mod handle_gpt_image_1_edit_sora; \ No newline at end of file diff --git a/crates/lib/files/images/Cargo.toml b/crates/lib/files/images/Cargo.toml index 585a625aff..2a90e47358 100644 --- a/crates/lib/files/images/Cargo.toml +++ b/crates/lib/files/images/Cargo.toml @@ -26,7 +26,7 @@ errors = { workspace = true} mimetypes = { workspace = true } # External -image = "0.25.6" +image = { version = "0.25.6" , features = ["jpeg", "png", "tiff", "webp"] } [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/lib/files/images/src/encoding/mod.rs b/crates/lib/files/images/src/encoding/mod.rs new file mode 100644 index 0000000000..c10c4775ea --- /dev/null +++ b/crates/lib/files/images/src/encoding/mod.rs @@ -0,0 +1 @@ +pub mod webp_bytes_to_png_bytes; \ No newline at end of file diff --git a/crates/lib/files/images/src/encoding/webp_bytes_to_png_bytes.rs b/crates/lib/files/images/src/encoding/webp_bytes_to_png_bytes.rs new file mode 100644 index 0000000000..66e96d06d6 --- /dev/null +++ b/crates/lib/files/images/src/encoding/webp_bytes_to_png_bytes.rs @@ -0,0 +1,21 @@ +use crate::error::ImagesError; +use image::{ImageFormat, ImageReader}; +use std::io::Cursor; + +pub fn webp_bytes_to_png_bytes( + webp_bytes: &[u8], +) -> Result, ImagesError> { + + let mut reader = ImageReader::new(Cursor::new(webp_bytes)); + + reader.set_format(ImageFormat::WebP); // Default to WebP format (as fallback). + + let image = reader + .with_guessed_format()? // NB: Can raise IoError. + .decode()?; + + let mut output_bytes: Vec = Vec::new(); + image.write_to(&mut Cursor::new(&mut output_bytes), ImageFormat::Png)?; + + Ok(output_bytes) +} \ No newline at end of file diff --git a/crates/lib/files/images/src/error.rs b/crates/lib/files/images/src/error.rs new file mode 100644 index 0000000000..e52a09c046 --- /dev/null +++ b/crates/lib/files/images/src/error.rs @@ -0,0 +1,31 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; + +#[derive(Debug)] +pub enum ImagesError { + IoError(std::io::Error), + ImageError(image::ImageError), +} + +impl Error for ImagesError {} + +impl Display for ImagesError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ImagesError::IoError(err) => write!(f, "IO Error: {}", err), + ImagesError::ImageError(err) => write!(f, "Image Error: {}", err), + } + } +} + +impl From for ImagesError { + fn from(err: std::io::Error) -> Self { + ImagesError::IoError(err) + } +} + +impl From for ImagesError { + fn from(err: image::ImageError) -> Self { + ImagesError::ImageError(err) + } +} diff --git a/crates/lib/files/images/src/lib.rs b/crates/lib/files/images/src/lib.rs index 40dca2f3b8..97a83f180d 100644 --- a/crates/lib/files/images/src/lib.rs +++ b/crates/lib/files/images/src/lib.rs @@ -23,6 +23,8 @@ // Re-export image to everyone importing this library pub use image; +pub mod encoding; +pub mod error; pub mod image_info; pub mod resize_image_file_preserving_aspect; pub mod resize_preserving_aspect; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_gpt_image_1_text_to_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_gpt_image_1_text_to_image.rs new file mode 100644 index 0000000000..6425b5760a --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/generate_gpt_image_1_text_to_image.rs @@ -0,0 +1,58 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GPT_IMAGE_1_TEXT_TO_IMAGE_PATH: &str = "/v1/generate/image/gpt_image_1_text_to_image"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateGptImage1TextToImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + /// Size of the image to generate. Default is Square. + pub image_size: Option, + + /// Number of images to generate. Default is one. + pub num_images: Option, + + /// Quality of the image to generate. Default is High. + pub image_quality: Option, +} + + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateGptImage1TextToImageImageSize { + Square, // 1:1, Default + Horizontal, + Vertical, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateGptImage1TextToImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateGptImage1TextToImageImageQuality { + Auto, + Low, + Medium, + High, // Default +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateGptImage1TextToImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs index 1d9185f67c..90f2d43e51 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs @@ -3,4 +3,5 @@ pub mod generate_flux_1_dev_text_to_image; pub mod generate_flux_1_schnell_text_to_image; pub mod generate_flux_pro_11_text_to_image; pub mod generate_flux_pro_11_ultra_text_to_image; +pub mod generate_gpt_image_1_text_to_image; pub mod remove_image_background; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index 830e16d075..ac7d821f17 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -141,6 +141,11 @@ use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageNumImages; use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageResponse; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::GenerateGptImage1TextToImageImageQuality; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::GenerateGptImage1TextToImageImageSize; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::GenerateGptImage1TextToImageNumImages; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::GenerateGptImage1TextToImageRequest; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::GenerateGptImage1TextToImageResponse; use artcraft_api_defs::generate::image::remove_image_background::RemoveImageBackgroundRequest; use artcraft_api_defs::generate::image::remove_image_background::RemoveImageBackgroundResponse; use artcraft_api_defs::generate::object::generate_hunyuan_2_0_image_to_3d::GenerateHunyuan20ImageTo3dRequest; @@ -237,6 +242,7 @@ use tokens::tokens::zs_voice_datasets::*; crate::http_server::endpoints::generate::image::generate_flux_1_dev_text_to_image_handler::generate_flux_1_dev_text_to_image_handler, crate::http_server::endpoints::generate::image::generate_flux_1_schnell_text_to_image_handler::generate_flux_1_schnell_text_to_image_handler, crate::http_server::endpoints::generate::image::generate_flux_pro_11_text_to_image_handler::generate_flux_pro_11_text_to_image_handler, + crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler, crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler, crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler, crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler, @@ -420,6 +426,11 @@ use tokens::tokens::zs_voice_datasets::*; BatchGetUserRatingError, BatchGetUserRatingQueryParams, BatchGetUserRatingResponse, + GenerateGptImage1TextToImageImageQuality, + GenerateGptImage1TextToImageImageSize, + GenerateGptImage1TextToImageNumImages, + GenerateGptImage1TextToImageRequest, + GenerateGptImage1TextToImageResponse, BatchInferenceJobStatusResponsePayload, BatchMediaFileInfo, BatchRequestDetailsResponse, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_gpt_image_1_text_to_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_gpt_image_1_text_to_image_handler.rs new file mode 100644 index 0000000000..4bc5e96ff6 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/generate_gpt_image_1_text_to_image_handler.rs @@ -0,0 +1,213 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageImageQuality, GenerateGptImage1TextToImageImageSize, GenerateGptImage1TextToImageNumImages, GenerateGptImage1TextToImageRequest, GenerateGptImage1TextToImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::enqueue_gpt_image_1_text_to_image_webhook::{enqueue_gpt_image_1_text_to_image_webhook, GptTextToImageByokArgs, GptTextToImageNumImages, GptTextToImageQuality, GptTextToImageSize}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Gpt Image 1 +#[utoipa::path( + post, + tag = "Generate Images", + path = "/v1/generate/image/gpt_image_1_text_to_image", + responses( + (status = 200, description = "Success", body = GenerateGptImage1TextToImageResponse), + ), + params( + ("request" = GenerateGptImage1TextToImageRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_gpt_image_1_text_to_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + let media_domain = get_media_domain(&http_request); + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let image_size = match request.image_size { + Some(GenerateGptImage1TextToImageImageSize::Square) => GptTextToImageSize::Square, + Some(GenerateGptImage1TextToImageImageSize::Horizontal) => GptTextToImageSize::Horizontal, + Some(GenerateGptImage1TextToImageImageSize::Vertical) => GptTextToImageSize::Vertical, + None => GptTextToImageSize::Square, // Default to Square + }; + + let num_images = match request.num_images { + Some(GenerateGptImage1TextToImageNumImages::One) => GptTextToImageNumImages::One, + Some(GenerateGptImage1TextToImageNumImages::Two) => GptTextToImageNumImages::Two, + Some(GenerateGptImage1TextToImageNumImages::Three) => GptTextToImageNumImages::Three, + Some(GenerateGptImage1TextToImageNumImages::Four) => GptTextToImageNumImages::Four, + None => GptTextToImageNumImages::One, // Default to One + }; + + let quality = match request.image_quality { + Some(quality) => match quality { + GenerateGptImage1TextToImageImageQuality::Auto => GptTextToImageQuality::Auto, + GenerateGptImage1TextToImageImageQuality::Low => GptTextToImageQuality::Low, + GenerateGptImage1TextToImageImageQuality::Medium => GptTextToImageQuality::Medium, + GenerateGptImage1TextToImageImageQuality::High => GptTextToImageQuality::High, + }, + None => GptTextToImageQuality::High, // Default to High + }; + + let openai_api_key = OpenAiApiKey::from_str(&server_state.openai.api_key); + + let args = GptTextToImageByokArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + num_images, + image_size, + quality, + openai_api_key: &openai_api_key, + }; + + let fal_result = enqueue_gpt_image_1_text_to_image_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_gpt_image_1_text_to_image_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::GptImage1), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(GenerateGptImage1TextToImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs index ebc12c1b8b..378ead72b8 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs @@ -3,4 +3,5 @@ pub mod generate_flux_1_dev_text_to_image_handler; pub mod generate_flux_1_schnell_text_to_image_handler; pub mod generate_flux_pro_11_text_to_image_handler; pub mod generate_flux_pro_11_ultra_text_to_image_handler; +pub mod generate_gpt_image_1_text_to_image_handler; pub mod remove_image_background_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs index 5c9ae75c23..26894b400e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs @@ -7,8 +7,10 @@ use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCat use enums::by_table::media_files::media_file_type::MediaFileType; use errors::AnyhowResult; use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use images::encoding::webp_bytes_to_png_bytes::webp_bytes_to_png_bytes; use images::image_info::image_info::ImageInfo; use log::info; +use mimetypes::mimetype_info::file_extension::FileExtension; use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::FalJobDetails; use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; @@ -32,18 +34,17 @@ pub async fn handle_image_payload( job: &FalJobDetails, server_state: &ServerState, ) -> AnyhowResult { - let image_value = payload.get("image") .ok_or_else(|| anyhow!("no `image` key in payload"))?; info!("Fal Image Payload: {:?}", image_value); - + let image: FalWebhookImage = serde_json::from_value(image_value.clone())?; let image_url = image.url .as_deref() .ok_or_else(|| anyhow!("no `url` in image payload"))?; - + //let mime_type = image.content_type // .as_deref() // .ok_or_else(|| anyhow!("no `content_type` in image payload"))?; @@ -51,10 +52,53 @@ pub async fn handle_image_payload( let file_bytes = http_download_url_to_bytes(image_url) .await .map_err(|e| anyhow!("Failed to download image: {:?}", e))?; - + let mimetype_info = MimetypeInfo::get_for_bytes(&file_bytes) .ok_or_else(|| anyhow!("Failed to get mimetype info"))?; - + + + info!("File type: {}, extension: {:?}", + mimetype_info.mime_type(), + mimetype_info.file_extension()); + + match mimetype_info.file_extension() { + Some(FileExtension::Webp) => { + info!("Artcraft can't handle WebP images yet; converting to PNG..."); + + let file_bytes = webp_bytes_to_png_bytes(&file_bytes)?; + + let mimetype_info = MimetypeInfo::get_for_bytes(&file_bytes) + .ok_or_else(|| anyhow!("Failed to get mimetype info"))?; + + info!("Updated file type: {}, extension: {:?}", + mimetype_info.mime_type(), + mimetype_info.file_extension()); + + upload_single_image_bytes( + job, + server_state, + &file_bytes, + mimetype_info, + ).await + } + _ => { + upload_single_image_bytes( + job, + server_state, + &file_bytes, + mimetype_info, + ).await + } + } +} + +async fn upload_single_image_bytes( + job: &FalJobDetails, + server_state: &ServerState, + file_bytes: &[u8], + mimetype_info: MimetypeInfo, +) -> AnyhowResult { + let mime_type = mimetype_info.mime_type(); let media_file_type = MediaFileType::try_from_mime_type(mime_type) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_images_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_images_payload.rs index 8398f5bcb9..0aa57c1c80 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_images_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_images_payload.rs @@ -7,8 +7,10 @@ use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCat use enums::by_table::media_files::media_file_type::MediaFileType; use errors::AnyhowResult; use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use images::encoding::webp_bytes_to_png_bytes::webp_bytes_to_png_bytes; use images::image_info::image_info::ImageInfo; use log::info; +use mimetypes::mimetype_info::file_extension::FileExtension; use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; use mysql_queries::queries::generic_inference::fal::get_inference_job_by_fal_id::FalJobDetails; use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; @@ -53,7 +55,7 @@ pub async fn handle_images_payload( Ok(maybe_media_token) } -pub async fn upload_image( +async fn upload_image( job: &FalJobDetails, server_state: &ServerState, image: &FalWebhookImage, @@ -73,11 +75,58 @@ pub async fn upload_image( let mimetype_info = MimetypeInfo::get_for_bytes(&file_bytes) .ok_or_else(|| anyhow!("Failed to get mimetype info"))?; + info!("File type: {}, extension: {:?}", + mimetype_info.mime_type(), + mimetype_info.file_extension()); + + match mimetype_info.file_extension() { + Some(FileExtension::Webp) => { + info!("Artcraft can't handle WebP images yet; converting to PNG..."); + + let file_bytes = webp_bytes_to_png_bytes(&file_bytes)?; + + let mimetype_info = MimetypeInfo::get_for_bytes(&file_bytes) + .ok_or_else(|| anyhow!("Failed to get mimetype info"))?; + + info!("Updated file type: {}, extension: {:?}", + mimetype_info.mime_type(), + mimetype_info.file_extension()); + + upload_image_bytes( + job, + server_state, + image, + &file_bytes, + mimetype_info, + ).await + } + _ => { + upload_image_bytes( + job, + server_state, + image, + &file_bytes, + mimetype_info, + ).await + } + } +} + +async fn upload_image_bytes( + job: &FalJobDetails, + server_state: &ServerState, + image: &FalWebhookImage, + file_bytes: &[u8], + mimetype_info: MimetypeInfo, +) -> AnyhowResult { + let mime_type = mimetype_info.mime_type(); let media_file_type = MediaFileType::try_from_mime_type(mime_type) .ok_or_else(|| anyhow!("Unsupported media file type: {}", mime_type))?; + info!("MediaFileType: {:?}", media_file_type); + let extension_with_period = mimetype_info.file_extension() .map(|ext| ext.extension_with_period()) .ok_or_else(|| anyhow!("Failed to get file extension from mimetype info"))?; @@ -117,4 +166,4 @@ pub async fn upload_image( info!("Image media file uploaded with token: {}", media_token); Ok(media_token) -} \ No newline at end of file +} diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index 6466b065cd..8700b54ff8 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -1,7 +1,9 @@ +use crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_1_dev_text_to_image_handler::generate_flux_1_dev_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_1_schnell_text_to_image_handler::generate_flux_1_schnell_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_pro_11_text_to_image_handler::generate_flux_pro_11_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler; +use crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler; use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_1_image_to_3d_handler::generate_hunyuan_2_1_image_to_3d_handler; @@ -14,7 +16,6 @@ use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::{web, App, Error, HttpResponse}; -use crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler; pub fn add_generate_routes (app: App) -> App where @@ -51,6 +52,10 @@ where .route(web::post().to(generate_flux_pro_11_ultra_text_to_image_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/gpt_image_1_text_to_image") + .route(web::post().to(generate_gpt_image_1_text_to_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/remove_background") .route(web::post().to(remove_image_background_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/gpt_image_1/edit_image/byok/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/gpt_image_1/edit_image/byok/mod.rs index 112dd45ac8..576cc80a84 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/gpt_image_1/edit_image/byok/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/gpt_image_1/edit_image/byok/mod.rs @@ -18,6 +18,22 @@ use crate::prelude::{Deserialize, FalRequest, Serialize}; } */ +/// This is the non-edit, text-to-image only endpoint +#[derive(Debug, Serialize, Deserialize)] +pub struct GptTextToImageRequest { + /// The prompt + pub prompt: String, + + pub image_size: String, + + pub num_images: u8, + + pub quality: String, + + pub openai_api_key: String, +} + +/// This is edit-only, image-to-image endpoint #[derive(Debug, Serialize, Deserialize)] pub struct GptEditImageRequest { /// The context images @@ -39,6 +55,10 @@ pub struct GptEditImageRequest { pub struct ImageOutput { } +pub fn gpt_text_to_image(params: GptTextToImageRequest) -> FalRequest { + FalRequest::new("fal-ai/gpt-image-1/text-to-image/byok", params) +} + pub fn gpt_edit_image(params: GptEditImageRequest) -> FalRequest { FalRequest::new("fal-ai/gpt-image-1/edit-image/byok", params) } diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts index 397d100841..30ed43f9ff 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts @@ -17,12 +17,14 @@ export interface EnqueueTextToImageRequest { prompt?: string; model?: ModelInfo | EnqueueTextToImageModel; aspect_ratio?: EnqueueTextToImageSize; + number_images?: number; } interface EnqueueTextToImageRawRequest { prompt?: string; model?: EnqueueTextToImageModel | string; // TODO: Shouldn't allow string aspect_ratio?: EnqueueTextToImageSize; + number_images?: number; } export enum EnqueueTextToImageModel { @@ -75,6 +77,10 @@ export const EnqueueTextToImage = async (request: EnqueueTextToImageRequest) : P mutableRequest.aspect_ratio = request.aspect_ratio; } + if (!!request.number_images) { + mutableRequest.number_images = request.number_images; + } + const result = await invoke("enqueue_text_to_image_command", { request: mutableRequest, });