diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs new file mode 100644 index 0000000000..f31e685162 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs @@ -0,0 +1,87 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::gemini_25_flash_image::edit::gemini_25_flash_image_edit; +use fal::prelude::fal_ai::gemini_25_flash_image::edit::Gemini25FlashImageEditInput; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Gemini25FlashEditArgs<'a, R: IntoUrl> { + // Request required + pub prompt: &'a str, + pub image_urls: Vec, + pub num_images: Gemini25FlashEditNumImages, + + // Fulfillment + pub webhook_url: R, + pub api_key: &'a FalApiKey, +} + +#[derive(Copy, Clone, Debug)] +pub enum Gemini25FlashEditNumImages { + One, + Two, + Three, + Four, +} + + +pub async fn enqueue_gemini_25_flash_edit_webhook( + args: Gemini25FlashEditArgs<'_, R> +) -> Result { + + let num_images = match args.num_images { + Gemini25FlashEditNumImages::One => 1, + Gemini25FlashEditNumImages::Two => 2, + Gemini25FlashEditNumImages::Three => 3, + Gemini25FlashEditNumImages::Four => 4, + }; + + let request = Gemini25FlashImageEditInput { + prompt: args.prompt.to_string(), + image_urls: args.image_urls, + // Constants + num_images: Some(num_images), + output_format: Some("png".to_string()), + }; + + let result = gemini_25_flash_image_edit(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::edit::enqueue_gemini_25_flash_edit_webhook::{enqueue_gemini_25_flash_edit_webhook, Gemini25FlashEditArgs, Gemini25FlashEditNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use test_data::web::image_urls::{GHOST_IMAGE_URL, TREX_SKELETON_IMAGE_URL}; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Gemini25FlashEditArgs { + image_urls: vec![ + GHOST_IMAGE_URL.to_string(), + TREX_SKELETON_IMAGE_URL.to_string(), + ], + num_images: Gemini25FlashEditNumImages::Two, + prompt: "add the ghost to the image of the t-rex skeleton, make it look spooky but friendly", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_gemini_25_flash_edit_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_edit_image_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gpt_image_1_edit_image_webhook.rs similarity index 93% rename from crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_edit_image_webhook.rs rename to crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gpt_image_1_edit_image_webhook.rs index 92b0497324..389f18dfb2 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/enqueue_gpt_image_1_edit_image_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gpt_image_1_edit_image_webhook.rs @@ -21,7 +21,7 @@ pub struct GptEditImageByokArgs<'a, V: IntoUrl> { } #[derive(Copy, Clone, Debug)] -pub enum GptEditImageSize{ +pub enum GptEditImageSize { Auto, Square, Horizontal, @@ -92,7 +92,7 @@ pub async fn enqueue_gpt_image_1_edit_image_webhook( mod tests { use crate::creds::fal_api_key::FalApiKey; use crate::creds::open_ai_api_key::OpenAiApiKey; - use crate::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::{enqueue_gpt_image_1_edit_image_webhook, GptEditImageByokArgs, GptEditImageNumImages, GptEditImageQuality, GptEditImageSize}; + use crate::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::{enqueue_gpt_image_1_edit_image_webhook, GptEditImageByokArgs, GptEditImageNumImages, GptEditImageQuality, GptEditImageSize}; use errors::AnyhowResult; use std::fs::read_to_string; use test_data::web::image_urls::{ERNEST_SCARED_STUPID_IMAGE_URL, GHOST_IMAGE_URL, GRASSY_HILL_TRANSPARENT_IMAGE_URL}; diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_qwen_image_edit_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_qwen_image_edit_webhook.rs index aa28a625ee..b3ee85f8d9 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_qwen_image_edit_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_qwen_image_edit_webhook.rs @@ -14,6 +14,13 @@ pub struct QwenImageEditArgs<'a, U: IntoUrl, R: IntoUrl> { pub num_images: Option, pub image_size: Option, + pub negative_prompt: Option, + + /// Acceleration level for image generation. + /// Options: 'none', 'regular'. Higher acceleration increases speed. + /// 'regular' balances speed and quality. Default value: "none" + pub acceleration: Option, + // Fulfillment pub webhook_url: R, pub api_key: &'a FalApiKey, @@ -109,6 +116,8 @@ mod tests { prompt: "put christmas lights on the tree, add snow to the mountains", num_images: Some(QwenImageEditNumImages::One), image_size: None, + negative_prompt: None, + acceleration: None, api_key: &api_key, webhook_url: "https://example.com/webhook", }; diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs index b9b58ed7d9..aaf27b4faf 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs @@ -1,3 +1,5 @@ pub mod enqueue_flux_pro_kontext_max_edit_webhook; +pub mod enqueue_gemini_25_flash_edit_webhook; +pub mod enqueue_gpt_image_1_edit_image_webhook; pub mod enqueue_qwen_image_edit_webhook; pub mod enqueue_seededit_v3_edit_webhook; \ No newline at end of file diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs index cd64404830..3c32e1f5e7 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs @@ -3,7 +3,6 @@ pub mod enqueue_flux_1_dev_text_to_image_webhook; pub mod enqueue_flux_1_schnell_text_to_image_webhook; pub mod enqueue_flux_pro_11_text_to_image_webhook; pub mod enqueue_flux_pro_11_ultra_text_to_image_webhook; -pub mod enqueue_gpt_image_1_edit_image_webhook; pub mod enqueue_gpt_image_1_text_to_image_webhook; pub mod infill; pub mod remove_background_rembg_webhook; diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs index 996259cba6..22d070b6f1 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs @@ -57,6 +57,8 @@ pub async fn enqueue_seedance_1_lite_image_to_video_webhook { + pub image_url: U, + pub webhook_url: V, + pub prompt: &'a str, + pub api_key: &'a FalApiKey, + pub camera_fixed: bool, + pub duration: Seedance1ProDuration, + pub resolution: Seedance1ProResolution, + pub seed: Option, +} + +#[derive(Copy, Clone, Debug)] +pub enum Seedance1ProDuration { + ThreeSeconds, + FourSeconds, + FiveSeconds, + SixSeconds, + SevenSeconds, + EightSeconds, + NineSeconds, + TenSeconds, + ElevenSeconds, + TwelveSeconds, +} + +#[derive(Copy, Clone, Debug)] +pub enum Seedance1ProResolution { + FourEightyP, // 480p + SevenTwentyP, // 720p + TenEightyP, // 1080p +} + +pub async fn enqueue_seedance_1_pro_image_to_video_webhook( + args: Seedance1ProArgs<'_, U, V> +) -> Result { + let duration = match args.duration { + Seedance1ProDuration::ThreeSeconds => Some("3".to_string()), + Seedance1ProDuration::FourSeconds => Some("4".to_string()), + Seedance1ProDuration::FiveSeconds => Some("5".to_string()), + Seedance1ProDuration::SixSeconds => Some("6".to_string()), + Seedance1ProDuration::SevenSeconds => Some("7".to_string()), + Seedance1ProDuration::EightSeconds => Some("8".to_string()), + Seedance1ProDuration::NineSeconds => Some("9".to_string()), + Seedance1ProDuration::TenSeconds => Some("10".to_string()), + Seedance1ProDuration::ElevenSeconds => Some("11".to_string()), + Seedance1ProDuration::TwelveSeconds => Some("12".to_string()), + }; + + let resolution = match args.resolution { + Seedance1ProResolution::FourEightyP => Some("480p".to_string()), + Seedance1ProResolution::SevenTwentyP => Some("720p".to_string()), + Seedance1ProResolution::TenEightyP => Some("1080p".to_string()), + }; + + let image_url = args.image_url.as_str().to_string(); + + let request = SeedanceImageToVideoProRequest { + image_url, + prompt: args.prompt.to_string(), + duration, + resolution, + // TODO: Add these later + camera_fixed: None, + // Static + enable_safety_checker: Some(false), + }; + + let result = seedance_v1_pro_image_to_video(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::{enqueue_seedance_1_pro_image_to_video_webhook, Seedance1ProArgs, Seedance1ProDuration, Seedance1ProResolution}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use test_data::web::image_urls::TALL_MOCHI_WITH_GLASSES_IMAGE_URL; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Seedance1ProArgs { + image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL, + prompt: "shiba in glasses runs to the lake and stands by the shore", + api_key: &api_key, + camera_fixed: false, + duration: Seedance1ProDuration::FiveSeconds, + resolution: Seedance1ProResolution::SevenTwentyP, + seed: None, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_seedance_1_pro_image_to_video_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_3_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_3_image_to_video_webhook.rs index c4d843b358..5a5ff02a26 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_3_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_veo_3_image_to_video_webhook.rs @@ -6,12 +6,13 @@ use fal::webhook::WebhookResponse; use reqwest::IntoUrl; pub struct Veo3Args<'a, U: IntoUrl, V: IntoUrl> { - pub prompt: &'a str, pub image_url: U, + pub prompt: &'a str, pub duration: Veo3Duration, - pub api_key: &'a FalApiKey, + pub aspect_ratio: Veo3AspectRatio, pub resolution: Veo3Resolution, pub generate_audio: bool, + pub api_key: &'a FalApiKey, pub webhook_url: V, } @@ -21,6 +22,14 @@ pub enum Veo3Duration { EightSeconds, } +#[derive(Copy, Clone, Debug)] +pub enum Veo3AspectRatio { + Default, + WideSixteenNine, + TallNineSixteen, + Square, +} + #[derive(Copy, Clone, Debug)] pub enum Veo3Resolution { Default, @@ -35,6 +44,13 @@ pub async fn enqueue_veo_3_image_to_video_webhook( Veo3Duration::Default => None, Veo3Duration::EightSeconds => Some("8s".to_string()), }; + + let aspect_ratio = match args.aspect_ratio { + Veo3AspectRatio::Default => None, + Veo3AspectRatio::WideSixteenNine => Some("16:9".to_string()), + Veo3AspectRatio::TallNineSixteen => Some("9:16".to_string()), + Veo3AspectRatio::Square => Some("1:1".to_string()), + }; let resolution= match args.resolution { Veo3Resolution::Default => None, @@ -47,6 +63,7 @@ pub async fn enqueue_veo_3_image_to_video_webhook( let request = ImageToVideoInput { image_url, prompt: args.prompt.to_string(), + aspect_ratio, resolution, duration, generate_audio: Some(args.generate_audio), @@ -64,28 +81,27 @@ pub async fn enqueue_veo_3_image_to_video_webhook( #[cfg(test)] mod tests { use crate::creds::fal_api_key::FalApiKey; - use crate::requests::webhook::video::enqueue_veo_3_image_to_video_webhook::{enqueue_veo_3_image_to_video_webhook, Veo3Args, Veo3Duration, Veo3Resolution}; + use crate::requests::webhook::video::enqueue_veo_3_image_to_video_webhook::{enqueue_veo_3_image_to_video_webhook, Veo3Args, Veo3AspectRatio, Veo3Duration, Veo3Resolution}; use errors::AnyhowResult; use std::fs::read_to_string; - use test_data::web::image_urls::JUNO_AT_LAKE_IMAGE_URL; + use test_data::web::image_urls::{JUNO_AT_LAKE_IMAGE_URL, TREX_SKELETON_IMAGE_URL}; #[tokio::test] #[ignore] async fn test() -> AnyhowResult<()> { - let image_url = JUNO_AT_LAKE_IMAGE_URL; - // XXX: Don't commit secrets! let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; let api_key = FalApiKey::from_str(&secret); let args = Veo3Args { - image_url: image_url, - prompt: "corgi at the lake. the corgi barks. camera pulls back to show a wider shot of the lake as the corgi jumps into the water. there are sounds of barking and splashing. cinematic, 8k, ultra wide shot", + image_url: TREX_SKELETON_IMAGE_URL, + prompt: "the t-rex skeleton starts walking towards the camera and roars", api_key: &api_key, duration: Veo3Duration::EightSeconds, - generate_audio: true, + aspect_ratio: Veo3AspectRatio::WideSixteenNine, resolution: Veo3Resolution::TenEightyP, + generate_audio: true, webhook_url: "https://example.com/webhook", }; diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/video/mod.rs index 05de033c3e..94ba9a5fb0 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/mod.rs @@ -2,6 +2,7 @@ pub mod enqueue_kling_16_pro_image_to_video_webhook; pub mod enqueue_kling_21_master_image_to_video_webhook; pub mod enqueue_kling_21_pro_image_to_video_webhook; pub mod enqueue_seedance_1_lite_image_to_video_webhook; +pub mod enqueue_seedance_1_pro_image_to_video_webhook; pub mod enqueue_veo_2_image_to_video_webhook; pub mod enqueue_veo_3_fast_image_to_video_webhook; pub mod enqueue_veo_3_image_to_video_webhook; \ No newline at end of file diff --git a/crates/api_clients/storyteller_client/src/generate/image/edit/gemini_25_flash_edit_image.rs b/crates/api_clients/storyteller_client/src/generate/image/edit/gemini_25_flash_edit_image.rs new file mode 100644 index 0000000000..37e6c3d292 --- /dev/null +++ b/crates/api_clients/storyteller_client/src/generate/image/edit/gemini_25_flash_edit_image.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageRequest, Gemini25FlashEditImageResponse, GEMINI_25_FLASH_EDIT_IMAGE_PATH}; + +pub async fn gemini_25_flash_edit_image( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Gemini25FlashEditImageRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + GEMINI_25_FLASH_EDIT_IMAGE_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs b/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs index ed56ffa74f..3b52212122 100644 --- a/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs +++ b/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs @@ -1,2 +1,3 @@ pub mod flux_pro_kontext_max_edit_image; +pub mod gemini_25_flash_edit_image; pub mod gpt_image_1_edit_image; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal_artcraft.rs index 930413909a..2049697a09 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/generic/handle_generic_bg_removal_artcraft.rs @@ -25,7 +25,6 @@ use artcraft_api_defs::generate::image::remove_image_background::RemoveImageBack use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; use idempotency::uuid::generate_random_uuid; use log::{error, info}; use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs index afef280035..33caf3446d 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs @@ -1,5 +1,6 @@ use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; use crate::core::commands::enqueue::image_edit::flux_kontext::handle_flux_kontext_edit::handle_flux_kontext_edit; +use crate::core::commands::enqueue::image_edit::gemini_25_flash::handle_gemini_25_flash_edit::handle_gemini_25_flash_edit; use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit::handle_gpt_image_1_edit; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; @@ -47,11 +48,20 @@ use tokens::tokens::media_files::MediaFileToken; #[derive(Deserialize, Debug, Copy, Clone)] #[serde(rename_all = "snake_case")] pub enum ContextualImageEditModel { - #[serde(rename = "gpt_image_1")] - GptImage1, - #[serde(rename = "flux_pro_kontext_max")] FluxProKontextMax, + + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, + + #[serde(rename = "gpt_image_1")] + GptImage1, + +// #[serde(rename = "qwen")] +// Qwen, +// +// #[serde(rename = "seededit_3")] +// SeedEdit3, } #[derive(Deserialize, Debug)] @@ -246,6 +256,20 @@ pub async fn handle_request( sora_task_queue, ).await? } + Some(ContextualImageEditModel::Gemini25Flash) => { + handle_gemini_25_flash_edit( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + fal_creds_manager, + fal_task_queue, + sora_creds_manager, + sora_task_queue, + ).await? + } Some(ContextualImageEditModel::GptImage1) => { handle_gpt_image_1_edit( request, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/flux_kontext/handle_flux_kontext_edit_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/flux_kontext/handle_flux_kontext_edit_artcraft.rs index 67787a1eab..6910f5f374 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/flux_kontext/handle_flux_kontext_edit_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/flux_kontext/handle_flux_kontext_edit_artcraft.rs @@ -23,7 +23,6 @@ use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::FluxProKontextMaxNumImages; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; use idempotency::uuid::generate_random_uuid; use log::{error, info}; use storyteller_client::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/handle_gemini_25_flash_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/handle_gemini_25_flash_edit.rs new file mode 100644 index 0000000000..06df36e462 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/handle_gemini_25_flash_edit.rs @@ -0,0 +1,59 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gemini_25_flash::handle_gemini_25_flash_edit_artcraft::handle_gemini_25_flash_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use log::info; +use tauri::AppHandle; + +pub async fn handle_gemini_25_flash_edit( + request: &EnqueueContextualEditImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough (TODO) + } + Provider::Sora => { + // Fallthrough + } + Provider::Artcraft => { + info!("Dispatching gemini 2.5 flash (edit) via Artcraft..."); + return handle_gemini_25_flash_edit_artcraft( + request, + app, + app_data_root, + app_env_configs, + storyteller_creds_manager + ).await; + } + } + } + + Err(InternalContextualEditImageError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/handle_gemini_25_flash_edit_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/handle_gemini_25_flash_edit_artcraft.rs new file mode 100644 index 0000000000..8b8510459a --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/handle_gemini_25_flash_edit_artcraft.rs @@ -0,0 +1,119 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest}; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest}; +use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use idempotency::uuid::generate_random_uuid; +use log::{error, info}; +use storyteller_client::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; +use storyteller_client::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use tauri::AppHandle; + +pub(super) const MAX_IMAGES: usize = 10; + +pub async fn handle_gemini_25_flash_edit_artcraft( + request: &EnqueueContextualEditImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalContextualEditImageError::NeedsStorytellerCredentials); + }, + }; + + info!("Calling Artcraft gemini 2.5 flash (edit) ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let num_images = match request.image_count { + None => None, + Some(1) => Some(Gemini25FlashEditImageNumImages::One), + Some(2) => Some(Gemini25FlashEditImageNumImages::Two), + Some(3) => Some(Gemini25FlashEditImageNumImages::Three), + Some(4) => Some(Gemini25FlashEditImageNumImages::Four), + Some(other) => { + return Err(InternalContextualEditImageError::InvalidNumberOfRequestedImages { + min: 1, + max: 4, + requested: other, + }); + }, + }; + + let mut media_tokens = Vec::with_capacity(10); + + if let Some(scene_image_media_token) = request.scene_image_media_token.clone() { + media_tokens.push(scene_image_media_token); + } + + if let Some(image_media_tokens) = request.image_media_tokens.as_ref() { + media_tokens.extend_from_slice(image_media_tokens); + } + + if media_tokens.len() > MAX_IMAGES { + return Err(InternalContextualEditImageError::InvalidNumberOfInputImages { + min: 1, + max: MAX_IMAGES as u32, + requested: media_tokens.len() as u32, + }); + } + + let request = Gemini25FlashEditImageRequest { + uuid_idempotency_token, + prompt: Some(request.prompt.clone()), + image_media_tokens: Some(media_tokens), + num_images, + image_quality: None, + }; + + let result = gemini_25_flash_edit_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + let job_id = match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft gpt-image-1. Job token: {}", + enqueued.inference_job_token); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft gpt-image-1: {:?}", err); + return Err(InternalContextualEditImageError::StorytellerError(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Artcraft, + model: Some(GenerationModel::Gemini25Flash), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::ImageGeneration, + }) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/mod.rs new file mode 100644 index 0000000000..092a12d1f2 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gemini_25_flash/mod.rs @@ -0,0 +1,2 @@ +pub (super) mod handle_gemini_25_flash_edit; +pub (super) mod handle_gemini_25_flash_edit_artcraft; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs index b1065239ad..16d75abc1b 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_edit_artcraft.rs @@ -22,7 +22,7 @@ use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; use idempotency::uuid::generate_random_uuid; use log::{error, info}; use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs index c89e043ec7..2fa1d875b7 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs @@ -1,4 +1,5 @@ pub mod enqueue_contextual_edit_image_command; pub mod errors; pub mod gpt_image_1; -mod flux_kontext; \ No newline at end of file +mod flux_kontext; +mod gemini_25_flash; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs index 90cc463bbe..fbda1877df 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs @@ -4,6 +4,7 @@ use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintE use crate::core::commands::enqueue::image_inpaint::flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint; use crate::core::commands::enqueue::image_inpaint::flux_pro_1_inpaint::handle_flux_pro_1_inpaint::handle_flux_pro_1_inpaint; use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint; +use crate::core::commands::enqueue::image_inpaint::gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType}; @@ -48,16 +49,26 @@ use tokens::tokens::media_files::MediaFileToken; #[derive(Deserialize, Debug, Copy, Clone)] #[serde(rename_all = "snake_case")] pub enum ImageInpaintModel { - // Mask-based inpainting models + /* + * Mask-based inpainting models + */ + #[serde(rename = "flux_dev_juggernaut")] FluxDevJuggernaut, + #[serde(rename = "flux_pro_1")] FluxPro1, - // Non-inpainting, instructive editing models - // NB: We're supporting these to keep the app simple and not over-complicate the javascript + /* + * Non-inpainting, instructive editing models + * NB: We're supporting these to keep the app simple and not over-complicate the javascript + */ + #[serde(rename = "flux_pro_kontext_max")] FluxProKontextMax, + + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, } #[derive(Deserialize, Debug)] @@ -250,7 +261,7 @@ pub async fn handle_request( ).await? } Some(ImageInpaintModel::FluxProKontextMax) => { - // Redirect to contextual image edit handler. + // TODO: Redirect to contextual image edit handler. handle_flux_pro_kontext_inpaint( request, app, @@ -264,6 +275,21 @@ pub async fn handle_request( sora_task_queue, ).await? } + Some(ImageInpaintModel::Gemini25Flash) => { + // TODO: Redirect to contextual image edit handler. + handle_gemini_25_flash_inpaint( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + fal_creds_manager, + fal_task_queue, + sora_creds_manager, + sora_task_queue, + ).await? + } }; let result = success_event diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs index 302c375900..4208133958 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs @@ -26,7 +26,6 @@ use artcraft_api_defs::generate::image::inpaint::flux_dev_juggernaut_inpaint_ima use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; use idempotency::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint_artcraft.rs index be605f4a37..e6abbeea40 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint_artcraft.rs @@ -26,7 +26,6 @@ use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{Flux use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; use idempotency::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_kontext_inpaint/handle_flux_pro_kontext_inpaint_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_kontext_inpaint/handle_flux_pro_kontext_inpaint_artcraft.rs index 9a2cd1e235..9709185f01 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_kontext_inpaint/handle_flux_pro_kontext_inpaint_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_kontext_inpaint/handle_flux_pro_kontext_inpaint_artcraft.rs @@ -28,7 +28,6 @@ use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{Flux use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; use idempotency::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint.rs new file mode 100644 index 0000000000..97fe41ebc3 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint.rs @@ -0,0 +1,60 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint_artcraft::handle_flux_pro_kontext_inpaint_artcraft; +use crate::core::commands::enqueue::image_inpaint::gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint_artcraft::handle_gemini_25_flash_inpaint_artcraft; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use log::info; +use tauri::AppHandle; + +pub async fn handle_gemini_25_flash_inpaint( + request: &EnqueueInpaintImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough (for now) + } + Provider::Sora => { + // Fallthrough + } + Provider::Artcraft => { + info!("Dispatching gemini 25 flash inpaint via Artcraft..."); + return handle_gemini_25_flash_inpaint_artcraft( + request, + app, + app_data_root, + app_env_configs, + storyteller_creds_manager + ).await; + } + } + } + + Err(InternalImageInpaintError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint_artcraft.rs new file mode 100644 index 0000000000..d9554431b5 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint_artcraft.rs @@ -0,0 +1,167 @@ +use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand; +use crate::core::commands::enqueue::image_bg_removal::errors::InternalBgRemovalError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest}; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest}; +use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest}; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use idempotency::uuid::generate_random_uuid; +use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; +use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; +use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; +use log::{error, info}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet; +use storyteller_client::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; +use storyteller_client::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image; +use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image; +use storyteller_client::generate::image::inpaint::flux_pro_1_inpaint_image::flux_pro_1_inpaint_image; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::media_files::upload_image_media_file_from_bytes::{upload_image_media_file_from_bytes, ImageType, UploadImageBytesArgs}; +use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use tauri::AppHandle; +use tokens::tokens::media_files::MediaFileToken; + +pub async fn handle_gemini_25_flash_inpaint_artcraft( + request: &EnqueueInpaintImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalImageInpaintError::NeedsStorytellerCredentials); + }, + }; + + let image_media_token = match &request.image_media_token { + Some(token) => token.clone(), + None => { + return Err(InternalImageInpaintError::NoSourceImageSpecified); + }, + }; + + info!("Calling Artcraft gemini 2.5 flash inpaint ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let num_images = match request.image_count { + None => None, + Some(1) => Some(Gemini25FlashEditImageNumImages::One), + Some(2) => Some(Gemini25FlashEditImageNumImages::Two), + Some(3) => Some(Gemini25FlashEditImageNumImages::Three), + Some(4) => Some(Gemini25FlashEditImageNumImages::Four), + Some(other) => { + return Err(InternalImageInpaintError::InvalidNumberOfRequestedImages { + min: 1, + max: 4, + requested: other, + }); + }, + }; + + let request = Gemini25FlashEditImageRequest { + uuid_idempotency_token, + prompt: Some(request.prompt.clone()), + image_media_tokens: Some(vec![image_media_token]), + num_images, + image_quality: None, + }; + + let result = gemini_25_flash_edit_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + let job_id = match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft gemini 2.5 flash inpaint. Job token: {}", + enqueued.inference_job_token); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft gemini 2.5 flash inpaint: {:?}", err); + return Err(InternalImageInpaintError::StorytellerError(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Artcraft, + model: Some(GenerationModel::Gemini25Flash), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::ImageInpaintEdit, + }) +} + +// async fn get_mask( +// request: &EnqueueInpaintImageCommand, +// app_env_configs: &AppEnvConfigs, +// storyteller_creds: &StorytellerCredentialSet, +// ) -> Result { +// +// if request.mask_image_media_token.is_some() && request.mask_image_raw_bytes.is_some() { +// return Err(InternalImageInpaintError::MaskMediaTokenAndBytesSupplied); +// } +// +// if let Some(token) = request.mask_image_media_token.as_ref() { +// return Ok(token.clone()); +// }; +// +// let image_bytes = request.mask_image_raw_bytes.as_ref() +// .ok_or(InternalImageInpaintError::NoMaskImageSpecified)?; +// +// let image_bytes = normalize_image_bytes_to_flux_mask(image_bytes) +// .map_err(|err| { +// error!("Failed to convert image bytes to png: {:?}", err); +// InternalImageInpaintError::CouldNotEncodeMask +// })?; +// +// info!("Uploading image media file from bytes..."); +// +// let result = upload_image_media_file_from_bytes(UploadImageBytesArgs { +// api_host: &app_env_configs.storyteller_host, +// maybe_creds: Some(&storyteller_creds), +// image_bytes: image_bytes.0, +// image_type: ImageType::Png, +// is_intermediate_system_file: true, +// }).await +// .map_err(|err| { +// error!("Failed to upload image media file: {:?}", err); +// InternalImageInpaintError::StorytellerError(err) +// })?; +// +// Ok(result.media_file_token) +// } diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/mod.rs new file mode 100644 index 0000000000..e5d386b210 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/mod.rs @@ -0,0 +1,2 @@ +pub (super) mod handle_gemini_25_flash_inpaint; +pub (super) mod handle_gemini_25_flash_inpaint_artcraft; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs index 7a0e988472..0e6e9ecf17 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs @@ -1,5 +1,6 @@ mod flux_dev_juggernaut_inpaint; mod flux_pro_1_inpaint; mod flux_pro_kontext_inpaint; +mod gemini_25_flash_inpaint; pub mod enqueue_image_inpaint_command; pub mod errors; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs b/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs index 3b7098aac3..b6a6435cd1 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs @@ -64,6 +64,7 @@ impl TaskEnqueueSuccess{ Some(GenerationModel::FluxPro11) => Some(TaskModelType::FluxPro11), Some(GenerationModel::FluxPro11Ultra) => Some(TaskModelType::FluxPro11Ultra), Some(GenerationModel::FluxProKontextMax) => Some(TaskModelType::FluxProKontextMax), + Some(GenerationModel::Gemini25Flash) => Some(TaskModelType::Gemini25Flash), Some(GenerationModel::GptImage1) => Some(TaskModelType::GptImage1), Some(GenerationModel::Recraft3) => Some(TaskModelType::Recraft3), Some(GenerationModel::Kling21Pro) => Some(TaskModelType::Kling21Pro), diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs index 9f18513463..d081b1ff56 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs @@ -1,4 +1,5 @@ use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::commands::enqueue::text_to_image::gemini_25_flash::handle_gemini_25_flash::handle_gemini_25_flash; use crate::core::commands::enqueue::text_to_image::generic::handle_image_artcraft::handle_image_artcraft; use crate::core::commands::enqueue::text_to_image::generic::handle_image_fal::handle_image_fal; use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1; @@ -32,7 +33,7 @@ use serde::{Deserialize, Serialize}; use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs}; use tauri::{AppHandle, State}; -/// This is used in the Tauri command bridge. +/// This is used in the Tauri command bridge. /// Don't change the serializations without coordinating with the frontend. #[derive(Deserialize, Debug, Copy, Clone)] #[serde(rename_all = "snake_case")] @@ -50,6 +51,9 @@ pub enum ImageModel { #[serde(rename = "recraft_3")] Recraft3, + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, + // Generic Midjourney model, version unknown. #[serde(rename = "midjourney")] Midjourney, @@ -277,6 +281,18 @@ pub async fn dispatch_request( None => { return Err(InternalImageError::NoModelSpecified); } + Some(ImageModel::Gemini25Flash) => { + return handle_gemini_25_flash( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + sora_creds_manager, + sora_task_queue, + ).await; + } Some(ImageModel::GptImage1) => { return handle_gpt_image_1( request, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash.rs new file mode 100644 index 0000000000..8f64d0ce72 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash.rs @@ -0,0 +1,66 @@ +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; +use crate::core::commands::enqueue::text_to_image::gemini_25_flash::handle_gemini_25_flash_artcraft::handle_gemini_25_flash_artcraft; +use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft; +use crate::core::commands::enqueue::text_to_image::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora; +use crate::core::commands::enqueue::text_to_image::internal_image_error::InternalImageError; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use log::{error, info}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use std::time::Duration; +use tauri::AppHandle; + +pub async fn handle_gemini_25_flash( + request: &EnqueueTextToImageRequest, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough (for now) + } + Provider::Sora => { + // Fallthrough + } + Provider::Artcraft => { + info!("Dispatching gemini 2.5 flash via Artcraft..."); + return handle_gemini_25_flash_artcraft( + request, + app_env_configs, + storyteller_creds_manager + ).await; + } + } + } + + Err(InternalImageError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs new file mode 100644 index 0000000000..4018891f34 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/handle_gemini_25_flash_artcraft.rs @@ -0,0 +1,98 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; +use crate::core::commands::enqueue::text_to_image::internal_image_error::InternalImageError; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest}; +use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageImageQuality, GenerateGptImage1TextToImageImageSize, GenerateGptImage1TextToImageNumImages, GenerateGptImage1TextToImageRequest}; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use idempotency::uuid::generate_random_uuid; +use log::{error, info}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use std::time::Duration; +use storyteller_client::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::generate::image::generate_gpt_image_1_text_to_image::generate_gpt_image_1_text_to_image; +use tauri::AppHandle; + +pub async fn handle_gemini_25_flash_artcraft( + request: &EnqueueTextToImageRequest, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalImageError::NeedsStorytellerCredentials); + }, + }; + + info!("Calling Artcraft gemini 2.5 flash ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + // let image_media_token = match &request.image_media_token { + // Some(token) => token.clone(), + // None => { + // return Err(InternalImageInpaintError::NoSourceImageSpecified); + // }, + // }; + + let num_images = match request.number_images{ + None => None, + Some(1) => Some(Gemini25FlashEditImageNumImages::One), + Some(2) => Some(Gemini25FlashEditImageNumImages::Two), + Some(3) => Some(Gemini25FlashEditImageNumImages::Three), + Some(4) => Some(Gemini25FlashEditImageNumImages::Four), + _ => Some(Gemini25FlashEditImageNumImages::One), // Default to one image if invalid number + }; + + let request = Gemini25FlashEditImageRequest { + uuid_idempotency_token, + prompt: request.prompt.clone(), + image_media_tokens: Some(vec![]), // TODO: Needs image media tokens. + num_images, + image_quality: None, + }; + + let result = gemini_25_flash_edit_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + let job_id = match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft gemini 2.5 flash. Job token: {}", + enqueued.inference_job_token); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft gemini 2.5 flash: {:?}", err); + return Err(InternalImageError::StorytellerError(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Artcraft, + model: Some(GenerationModel::GptImage1), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::ImageGeneration, + }) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/mod.rs new file mode 100644 index 0000000000..0a64fffd65 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/gemini_25_flash/mod.rs @@ -0,0 +1,2 @@ +pub (crate) mod handle_gemini_25_flash_artcraft; +pub (super) mod handle_gemini_25_flash; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_artcraft.rs index 80de227e3b..e86482a615 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_artcraft.rs @@ -57,6 +57,7 @@ pub async fn handle_image_artcraft( return Err(InternalImageError::NoModelSpecified); } Some( + ImageModel::Gemini25Flash | ImageModel::GptImage1 | ImageModel::Midjourney ) => { diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_fal.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_fal.rs index 6bcbc5d1e0..2c7ba26c03 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_fal.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/generic/handle_image_fal.rs @@ -57,7 +57,8 @@ pub async fn handle_image_fal( return Err(InternalImageError::AnyhowError(anyhow!("wrong logic: another branch should handle this: {:?}", request.model))); } Some( - ImageModel::Flux1Dev | + ImageModel::Gemini25Flash | + ImageModel::Flux1Dev | ImageModel::Flux1Schnell | ImageModel::FluxPro11 ) => { diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/mod.rs index db3e2e58f8..839f6ca98b 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/mod.rs @@ -1,5 +1,6 @@ pub mod enqueue_text_to_image_command; pub mod generic; pub mod internal_image_error; +mod gemini_25_flash; mod gpt_image_1; mod midjourney; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/events/generation_events/common.rs b/crates/desktop/artcraft/src/core/events/generation_events/common.rs index 77ff0d04ac..9afa70d177 100644 --- a/crates/desktop/artcraft/src/core/events/generation_events/common.rs +++ b/crates/desktop/artcraft/src/core/events/generation_events/common.rs @@ -24,10 +24,14 @@ pub enum GenerationModel { FluxPro11, #[serde(rename = "flux_pro_1.1_ultra")] FluxPro11Ultra, + #[serde(rename = "flux_pro_kontext_max")] FluxProKontextMax, #[serde(rename = "gpt_image_1")] GptImage1, + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, + #[serde(rename = "hunyuan_3d_2_0")] Hunyuan3d2_0, #[serde(rename = "hunyuan_3d_2_1")] diff --git a/crates/lib/test_data/src/web/image_urls.rs b/crates/lib/test_data/src/web/image_urls.rs index e7b83a2812..d4eb40880a 100644 --- a/crates/lib/test_data/src/web/image_urls.rs +++ b/crates/lib/test_data/src/web/image_urls.rs @@ -32,3 +32,6 @@ pub const TALL_MOCHI_WITH_GLASSES_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/ // media token: m_7x18h6qnyay4a42kyyyw2xctasx5k9 pub const TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/r/e/j/b/p/rejbp61m56hqx4a498xh7j47vkjd7jnh/upload_rejbp61m56hqx4a498xh7j47vkjd7jnh.png"; + +// media token: m_77cpdyaby60z0qv9g6tv67gw2a1337 +pub const TREX_SKELETON_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/f/4/j/5/3/f4j53296sc35wqs3csgw4j4vvjf5yyxe/image_f4j53296sc35wqs3csgw4j4vvjf5yyxe.png"; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/gemini_25_flash_edit_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/gemini_25_flash_edit_image.rs new file mode 100644 index 0000000000..5fceacff3b --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/gemini_25_flash_edit_image.rs @@ -0,0 +1,50 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GEMINI_25_FLASH_EDIT_IMAGE_PATH: &str = "/v1/generate/image/edit/gemini_25_flash"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Gemini25FlashEditImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + // TODO: OneOf for image_urls or image_media_tokens + /// Image media tokens to include in the editing context. + pub image_media_tokens: Option>, + + /// Number of images to generate. Default is one. + pub num_images: Option, + + /// Quality of the image to generate. Default is High. + pub image_quality: Option, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Gemini25FlashEditImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Gemini25FlashEditImageImageQuality { + Auto, + Low, + Medium, + High, // Default +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Gemini25FlashEditImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs index e3014dfd6a..8247e9387a 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs @@ -1,3 +1,6 @@ pub mod flux_pro_kontext_max_edit_image; pub mod gpt_image_1_edit_image; +pub mod gemini_25_flash_edit_image; +pub mod qwen_edit_image; +pub mod seededit_3_edit_image; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/qwen_edit_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/qwen_edit_image.rs new file mode 100644 index 0000000000..b65669591d --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/qwen_edit_image.rs @@ -0,0 +1,51 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const QWEN_EDIT_IMAGE_PATH: &str = "/v1/generate/image/edit/qwen"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct QwenEditImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// The image we're editing. + pub image_media_token: MediaFileToken, + + /// Text prompt to generate the image from. + pub prompt: Option, + + /// Text prompt to generate the image from. + pub negative_prompt: Option, + + /// Acceleration level for image generation. + /// Options: 'none', 'regular'. Higher acceleration increases speed. + /// 'regular' balances speed and quality. Default value: "none" + pub acceleration: Option, + + /// Number of images to generate. Default is one. + pub num_images: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub enum QwenEditImageAcceleration { + None, + Regular, + High, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum QwenEditImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct QwenEditImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/seededit_3_edit_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/seededit_3_edit_image.rs new file mode 100644 index 0000000000..0551f34e1d --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/seededit_3_edit_image.rs @@ -0,0 +1,24 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const SEEDEDIT_3_EDIT_IMAGE_PATH: &str = "/v1/generate/image/edit/seededit_3"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct SeedEdit3EditImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// The image we're editing. + pub image_media_token: MediaFileToken, + + /// Text prompt to generate the image from. + pub prompt: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct SeedEdit3EditImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs new file mode 100644 index 0000000000..f14680f28b --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs @@ -0,0 +1,54 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GENERATE_SEEDANCE_1_0_PRO_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/seedance_1.0_pro_image_to_video"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateSeedance10ProImageToVideoRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Source of the image file to convert to video. + /// It must be an image. + pub media_file_token: Option, + + /// Optional text prompt. + pub prompt: Option, + + /// Optional: aspect ratio of the generated video. + pub resolution: Option, + + /// Optional: aspect ratio of the generated video. + pub duration: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateSeedance10ProResolution { + FourEightyP, + SevenTwentyP, + TenEightyP, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateSeedance10ProDuration { + ThreeSeconds, + FourSeconds, + FiveSeconds, + SixSeconds, + SevenSeconds, + EightSeconds, + NineSeconds, + TenSeconds, + ElevenSeconds, + TwelveSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateSeedance10ProImageToVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_veo_3_fast_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_veo_3_fast_image_to_video.rs new file mode 100644 index 0000000000..80af0d6ced --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_veo_3_fast_image_to_video.rs @@ -0,0 +1,47 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GENERATE_VEO_3_FAST_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/veo_3_fast_image_to_video"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateVeo3FastImageToVideoRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Source of the image file to convert to video. + /// It must be an image. + pub media_file_token: Option, + + /// Optional text prompt. + pub prompt: Option, + + /// Optional: resolution of the generated video. + pub resolution: Option, + + /// Optional: aspect ratio of the generated video. + pub duration: Option, + + /// Optional: Generate an audio track along with the video + pub generate_audio: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateVeo3FastResolution { + SevenTwentyP, // 720p + TenEightyP, // 1080p +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateVeo3FastDuration { + EightSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateVeo3FastImageToVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_veo_3_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_veo_3_image_to_video.rs new file mode 100644 index 0000000000..9325cf1350 --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_veo_3_image_to_video.rs @@ -0,0 +1,65 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GENERATE_VEO_3_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/veo_3_image_to_video"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateVeo3ImageToVideoRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Source of the image file to convert to video. + /// It must be an image. + pub media_file_token: Option, + + /// Optional text prompt. + pub prompt: Option, + + /// Optional: aspect ratio of the generated video. + pub aspect_ratio: Option, + + /// Optional: resolution of the generated video. + pub resolution: Option, + + /// Optional: aspect ratio of the generated video. + pub duration: Option, + + /// Optional: Generate an audio track along with the video + pub generate_audio: Option, + + // /// Optional. + // pub enhance_prompt: Option, + // /// Optional. Whether to automatically attempt to fix prompts that fail + // /// content policy or other validation checks by rewriting them + // /// Default value: true + // pub auto_fix: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateVeo3AspectRatio { + WideSixteenNine, // 16:9 + TallNineSixteen, // 9:16 + Square, // 1:1 +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateVeo3Resolution { + SevenTwentyP, // 720p + TenEightyP, // 1080p +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateVeo3Duration { + EightSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateVeo3ImageToVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs index 9dae98d730..1b61842a11 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs @@ -2,4 +2,7 @@ pub mod generate_kling_1_6_pro_image_to_video; pub mod generate_kling_2_1_master_image_to_video; pub mod generate_kling_2_1_pro_image_to_video; pub mod generate_seedance_1_0_lite_image_to_video; -pub mod generate_veo_2_image_to_video; \ No newline at end of file +pub mod generate_seedance_1_0_pro_image_to_video; +pub mod generate_veo_2_image_to_video; +pub mod generate_veo_3_fast_image_to_video; +pub mod generate_veo_3_image_to_video; \ No newline at end of file diff --git a/crates/schema/public/enums/src/common/model_type.rs b/crates/schema/public/enums/src/common/model_type.rs index 5a6a186fc4..449fc896f9 100644 --- a/crates/schema/public/enums/src/common/model_type.rs +++ b/crates/schema/public/enums/src/common/model_type.rs @@ -31,6 +31,12 @@ pub enum ModelType { GptImage1, #[serde(rename = "recraft_3")] Recraft3, + #[serde(rename = "seededit_3")] + SeedEdit3, + #[serde(rename = "qwen")] + Qwen, + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, /// Midjourney without distinguishing a model type or version #[serde(rename = "midjourney")] @@ -63,8 +69,14 @@ pub enum ModelType { Kling21Master, #[serde(rename = "seedance_1p0_lite")] Seedance10Lite, + #[serde(rename = "seedance_1p0_pro")] + Seedance10Pro, #[serde(rename = "veo_2")] Veo2, + #[serde(rename = "veo_3")] + Veo3, + #[serde(rename = "veo_3_fast")] + Veo3Fast, // 3D Object generation models #[serde(rename = "hunyuan_3d_2p0")] @@ -92,6 +104,9 @@ impl ModelType { Self::FluxProKontextMax => "flux_pro_kontext_max", Self::GptImage1 => "gpt_image_1", Self::Recraft3 => "recraft_3", + Self::SeedEdit3 => "seededit_3", + Self::Qwen => "qwen", + Self::Gemini25Flash => "gemini_25_flash", Self::Midjourney => "midjourney", Self::MidjourneyV6 => "midjourney_v6", Self::MidjourneyV6p1 => "midjourney_v6p1", @@ -106,7 +121,10 @@ impl ModelType { Self::Kling21Pro => "kling_2p1_pro", Self::Kling21Master => "kling_2p1_master", Self::Seedance10Lite => "seedance_1p0_lite", + Self::Seedance10Pro => "seedance_1p0_pro", Self::Veo2 => "veo_2", + Self::Veo3 => "veo_3", + Self::Veo3Fast => "veo_3_fast", // 3D Object generation models Self::Hunyuan3d2_0 => "hunyuan_3d_2p0", @@ -126,6 +144,9 @@ impl ModelType { "flux_pro_kontext_max" => Ok(Self::FluxProKontextMax), "gpt_image_1" => Ok(Self::GptImage1), "recraft_3" => Ok(Self::Recraft3), + "seededit_3" => Ok(Self::SeedEdit3), + "qwen" => Ok(Self::Qwen), + "gemini_25_flash" => Ok(Self::Gemini25Flash), "midjourney" => Ok(Self::Midjourney), "midjourney_v6" => Ok(Self::MidjourneyV6), "midjourney_v6p1" => Ok(Self::MidjourneyV6p1), @@ -140,7 +161,10 @@ impl ModelType { "kling_2p1_pro" => Ok(Self::Kling21Pro), "kling_2p1_master" => Ok(Self::Kling21Master), "seedance_1p0_lite" => Ok(Self::Seedance10Lite), + "seedance_1p0_pro" => Ok(Self::Seedance10Pro), "veo_2" => Ok(Self::Veo2), + "veo_3" => Ok(Self::Veo3), + "veo_3_fast" => Ok(Self::Veo3Fast), // 3D Object generation models "hunyuan_3d_2p0" => Ok(Self::Hunyuan3d2_0), @@ -164,6 +188,9 @@ impl ModelType { Self::FluxProKontextMax, Self::GptImage1, Self::Recraft3, + Self::SeedEdit3, + Self::Qwen, + Self::Gemini25Flash, Self::Midjourney, Self::MidjourneyV6, Self::MidjourneyV6p1, @@ -178,7 +205,10 @@ impl ModelType { Self::Kling21Pro, Self::Kling21Master, Self::Seedance10Lite, + Self::Seedance10Pro, Self::Veo2, + Self::Veo3, + Self::Veo3Fast, // 3D Object generation models Self::Hunyuan3d2_0, @@ -207,6 +237,9 @@ mod tests { assert_serialization(ModelType::FluxProKontextMax, "flux_pro_kontext_max"); assert_serialization(ModelType::GptImage1, "gpt_image_1"); assert_serialization(ModelType::Recraft3, "recraft_3"); + assert_serialization(ModelType::SeedEdit3, "seededit_3"); + assert_serialization(ModelType::Qwen, "qwen"); + assert_serialization(ModelType::Gemini25Flash, "gemini_25_flash"); assert_serialization(ModelType::Midjourney, "midjourney"); assert_serialization(ModelType::MidjourneyV6, "midjourney_v6"); assert_serialization(ModelType::MidjourneyV6p1, "midjourney_v6p1"); @@ -220,7 +253,10 @@ mod tests { assert_serialization(ModelType::Kling21Pro, "kling_2p1_pro"); assert_serialization(ModelType::Kling21Master, "kling_2p1_master"); assert_serialization(ModelType::Seedance10Lite, "seedance_1p0_lite"); + assert_serialization(ModelType::Seedance10Pro, "seedance_1p0_pro"); assert_serialization(ModelType::Veo2, "veo_2"); + assert_serialization(ModelType::Veo3, "veo_3"); + assert_serialization(ModelType::Veo3Fast, "veo_3_fast"); // 3D Object generation models assert_serialization(ModelType::Hunyuan3d2_0, "hunyuan_3d_2p0"); assert_serialization(ModelType::Hunyuan3d2_1, "hunyuan_3d_2p1"); @@ -238,6 +274,9 @@ mod tests { assert_eq!(ModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max"); assert_eq!(ModelType::GptImage1.to_str(), "gpt_image_1"); assert_eq!(ModelType::Recraft3.to_str(), "recraft_3"); + assert_eq!(ModelType::SeedEdit3.to_str(), "seededit_3"); + assert_eq!(ModelType::Qwen.to_str(), "qwen"); + assert_eq!(ModelType::Gemini25Flash.to_str(), "gemini_25_flash"); assert_eq!(ModelType::Midjourney.to_str(), "midjourney"); assert_eq!(ModelType::MidjourneyV6.to_str(), "midjourney_v6"); assert_eq!(ModelType::MidjourneyV6p1.to_str(), "midjourney_v6p1"); @@ -252,7 +291,10 @@ mod tests { assert_eq!(ModelType::Kling21Pro.to_str(), "kling_2p1_pro"); assert_eq!(ModelType::Kling21Master.to_str(), "kling_2p1_master"); assert_eq!(ModelType::Seedance10Lite.to_str(), "seedance_1p0_lite"); + assert_eq!(ModelType::Seedance10Pro.to_str(), "seedance_1p0_pro"); assert_eq!(ModelType::Veo2.to_str(), "veo_2"); + assert_eq!(ModelType::Veo3.to_str(), "veo_3"); + assert_eq!(ModelType::Veo3Fast.to_str(), "veo_3_fast"); // 3D Object generation models assert_eq!(ModelType::Hunyuan3d2_0.to_str(), "hunyuan_3d_2p0"); @@ -271,6 +313,9 @@ mod tests { assert_eq!(ModelType::from_str("flux_pro_kontext_max").unwrap(), ModelType::FluxProKontextMax); assert_eq!(ModelType::from_str("gpt_image_1").unwrap(), ModelType::GptImage1); assert_eq!(ModelType::from_str("recraft_3").unwrap(), ModelType::Recraft3); + assert_eq!(ModelType::from_str("seededit_3").unwrap(), ModelType::SeedEdit3); + assert_eq!(ModelType::from_str("qwen").unwrap(), ModelType::Qwen); + assert_eq!(ModelType::from_str("gemini_25_flash").unwrap(), ModelType::Gemini25Flash); assert_eq!(ModelType::from_str("midjourney").unwrap(), ModelType::Midjourney); assert_eq!(ModelType::from_str("midjourney_v6").unwrap(), ModelType::MidjourneyV6); assert_eq!(ModelType::from_str("midjourney_v6p1").unwrap(), ModelType::MidjourneyV6p1); @@ -284,7 +329,10 @@ mod tests { assert_eq!(ModelType::from_str("kling_2p1_pro").unwrap(), ModelType::Kling21Pro); assert_eq!(ModelType::from_str("kling_2p1_master").unwrap(), ModelType::Kling21Master); assert_eq!(ModelType::from_str("seedance_1p0_lite").unwrap(), ModelType::Seedance10Lite); + assert_eq!(ModelType::from_str("seedance_1p0_pro").unwrap(), ModelType::Seedance10Pro); assert_eq!(ModelType::from_str("veo_2").unwrap(), ModelType::Veo2); + assert_eq!(ModelType::from_str("veo_3").unwrap(), ModelType::Veo3); + assert_eq!(ModelType::from_str("veo_3_fast").unwrap(), ModelType::Veo3Fast); // 3D Object generation models assert_eq!(ModelType::from_str("hunyuan_3d_2p0").unwrap(), ModelType::Hunyuan3d2_0); assert_eq!(ModelType::from_str("hunyuan_3d_2p1").unwrap(), ModelType::Hunyuan3d2_1); @@ -293,7 +341,7 @@ mod tests { #[test] fn all_variants() { let mut variants = ModelType::all_variants(); - assert_eq!(variants.len(), 24); + assert_eq!(variants.len(), 30); // Image models assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell)); @@ -304,6 +352,9 @@ mod tests { assert_eq!(variants.pop_first(), Some(ModelType::FluxProKontextMax)); assert_eq!(variants.pop_first(), Some(ModelType::GptImage1)); assert_eq!(variants.pop_first(), Some(ModelType::Recraft3)); + assert_eq!(variants.pop_first(), Some(ModelType::SeedEdit3)); + assert_eq!(variants.pop_first(), Some(ModelType::Qwen)); + assert_eq!(variants.pop_first(), Some(ModelType::Gemini25Flash)); assert_eq!(variants.pop_first(), Some(ModelType::Midjourney)); assert_eq!(variants.pop_first(), Some(ModelType::MidjourneyV6)); assert_eq!(variants.pop_first(), Some(ModelType::MidjourneyV6p1)); @@ -317,7 +368,10 @@ mod tests { assert_eq!(variants.pop_first(), Some(ModelType::Kling21Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Kling21Master)); assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Lite)); + assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Veo2)); + assert_eq!(variants.pop_first(), Some(ModelType::Veo3)); + assert_eq!(variants.pop_first(), Some(ModelType::Veo3Fast)); // 3D Object generation models assert_eq!(variants.pop_first(), Some(ModelType::Hunyuan3d2_0)); assert_eq!(variants.pop_first(), Some(ModelType::Hunyuan3d2_1)); diff --git a/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs b/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs index 7b67667f53..79ce2ce293 100644 --- a/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs +++ b/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs @@ -26,6 +26,8 @@ pub enum TaskModelType { FluxPro11Ultra, #[serde(rename = "flux_pro_kontext_max")] FluxProKontextMax, + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, #[serde(rename = "gpt_image_1")] GptImage1, #[serde(rename = "recraft_3")] @@ -71,6 +73,7 @@ impl TaskModelType { Self::FluxPro11 => "flux_pro_1.1", Self::FluxPro11Ultra => "flux_pro_1.1_ultra", Self::FluxProKontextMax => "flux_pro_kontext_max", + Self::Gemini25Flash => "gemini_25_flash", Self::GptImage1 => "gpt_image_1", Self::Recraft3 => "recraft_3", Self::Midjourney => "midjourney", @@ -96,6 +99,7 @@ impl TaskModelType { "flux_pro_1.1" => Ok(Self::FluxPro11), "flux_pro_1.1_ultra" => Ok(Self::FluxPro11Ultra), "flux_pro_kontext_max" => Ok(Self::FluxProKontextMax), + "gemini_25_flash" => Ok(Self::Gemini25Flash), "gpt_image_1" => Ok(Self::GptImage1), "recraft_3" => Ok(Self::Recraft3), "midjourney" => Ok(Self::Midjourney), @@ -124,6 +128,7 @@ impl TaskModelType { Self::FluxPro11, Self::FluxPro11Ultra, Self::FluxProKontextMax, + Self::Gemini25Flash, Self::GptImage1, Self::Recraft3, Self::Midjourney, @@ -158,6 +163,7 @@ mod tests { assert_serialization(TaskModelType::FluxPro11, "flux_pro_1.1"); assert_serialization(TaskModelType::FluxPro11Ultra, "flux_pro_1.1_ultra"); assert_serialization(TaskModelType::FluxProKontextMax, "flux_pro_kontext_max"); + assert_serialization(TaskModelType::Gemini25Flash, "gemini_25_flash"); assert_serialization(TaskModelType::GptImage1, "gpt_image_1"); assert_serialization(TaskModelType::Recraft3, "recraft_3"); assert_serialization(TaskModelType::Midjourney, "midjourney"); @@ -182,6 +188,7 @@ mod tests { assert_eq!(TaskModelType::FluxPro11.to_str(), "flux_pro_1.1"); assert_eq!(TaskModelType::FluxPro11Ultra.to_str(), "flux_pro_1.1_ultra"); assert_eq!(TaskModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max"); + assert_eq!(TaskModelType::Gemini25Flash.to_str(), "gemini_25_flash"); assert_eq!(TaskModelType::GptImage1.to_str(), "gpt_image_1"); assert_eq!(TaskModelType::Recraft3.to_str(), "recraft_3"); assert_eq!(TaskModelType::Midjourney.to_str(), "midjourney"); @@ -206,6 +213,7 @@ mod tests { assert_eq!(TaskModelType::from_str("flux_pro_1.1").unwrap(), TaskModelType::FluxPro11); assert_eq!(TaskModelType::from_str("flux_pro_1.1_ultra").unwrap(), TaskModelType::FluxPro11Ultra); assert_eq!(TaskModelType::from_str("flux_pro_kontext_max").unwrap(), TaskModelType::FluxProKontextMax); + assert_eq!(TaskModelType::from_str("gemini_25_flash").unwrap(), TaskModelType::Gemini25Flash); assert_eq!(TaskModelType::from_str("gpt_image_1").unwrap(), TaskModelType::GptImage1); assert_eq!(TaskModelType::from_str("recraft_3").unwrap(), TaskModelType::Recraft3); assert_eq!(TaskModelType::from_str("midjourney").unwrap(), TaskModelType::Midjourney); @@ -223,7 +231,7 @@ mod tests { #[test] fn all_variants() { let mut variants = TaskModelType::all_variants(); - assert_eq!(variants.len(), 17); + assert_eq!(variants.len(), 18); // Image models assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Schnell)); @@ -232,6 +240,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11)); assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11Ultra)); assert_eq!(variants.pop_first(), Some(TaskModelType::FluxProKontextMax)); + assert_eq!(variants.pop_first(), Some(TaskModelType::Gemini25Flash)); assert_eq!(variants.pop_first(), Some(TaskModelType::GptImage1)); assert_eq!(variants.pop_first(), Some(TaskModelType::Recraft3)); assert_eq!(variants.pop_first(), Some(TaskModelType::Midjourney)); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs new file mode 100644 index 0000000000..99675b4bc4 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs @@ -0,0 +1,258 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest, Gemini25FlashEditImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_gemini_25_flash_edit_webhook::{enqueue_gemini_25_flash_edit_webhook, Gemini25FlashEditArgs, Gemini25FlashEditNumImages}; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::GptEditImageNumImages; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::{GptEditImageByokArgs, GptEditImageQuality, GptEditImageSize}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Gemini 2.5 Flash Image Editing +#[utoipa::path( + post, + tag = "Generate Images (Edit)", + path = "/v1/generate/image/edit/gemini_25_flash", + responses( + (status = 200, description = "Success", body = Gemini25FlashEditImageResponse), + ), + params( + ("request" = Gemini25FlashEditImageRequest, description = "Payload for Request"), + ) +)] +pub async fn gemini_25_flash_edit_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + let tokens = match request.image_media_tokens.as_ref() { + Some(tokens) => tokens, + None => { + warn!("No image media tokens provided"); + return Err(CommonWebError::BadInputWithSimpleMessage("No image media tokens provided".to_string())); + } + }; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if media_files.len() != tokens.len() { + warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), tokens.len()); + return Err(CommonWebError::BadInputWithSimpleMessage( + format!("Not all media files could be found. Media files found: {}, tokens provided: {}", + media_files.len(), tokens.len()))); + } + + let media_domain = get_media_domain(&http_request); + + let image_urls = media_files.iter() + .map(|file| { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &file.public_bucket_directory_hash, + file.maybe_public_bucket_prefix.as_deref(), + file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }) + .collect::>(); + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let num_images = match request.num_images { + Some(Gemini25FlashEditImageNumImages::One) => Gemini25FlashEditNumImages::One, + Some(Gemini25FlashEditImageNumImages::Two) => Gemini25FlashEditNumImages::Two, + Some(Gemini25FlashEditImageNumImages::Three) => Gemini25FlashEditNumImages::Three, + Some(Gemini25FlashEditImageNumImages::Four) => Gemini25FlashEditNumImages::Four, + None => Gemini25FlashEditNumImages::One, // Default to One + }; + + let args = Gemini25FlashEditArgs { + image_urls, + prompt: request.prompt.as_deref().unwrap_or(""), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + num_images, + }; + + let fal_result = enqueue_gemini_25_flash_edit_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_gpt_image_1_edit_image_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Gemini25Flash), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: media_files.iter().map(|media_file| { + PromptContextItem { + media_token: media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgref, + } + }).collect(), + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Gemini25FlashEditImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gpt_image_1_edit_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gpt_image_1_edit_image_handler.rs index f25bba9f66..cefead5904 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gpt_image_1_edit_image_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gpt_image_1_edit_image_handler.rs @@ -9,14 +9,15 @@ use actix_web::web::Json; use actix_web::{web, HttpRequest}; use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest, GptImage1EditImageResponse}; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; use enums::by_table::prompts::prompt_type::PromptType; use enums::common::generation_provider::GenerationProvider; use enums::common::model_type::ModelType; use enums::common::visibility::Visibility; use fal_client::creds::open_ai_api_key::OpenAiApiKey; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageNumImages; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::{GptEditImageByokArgs, GptEditImageQuality, GptEditImageSize}; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::GptEditImageNumImages; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1_edit_image_webhook::{GptEditImageByokArgs, GptEditImageQuality, GptEditImageSize}; use http_server_common::request::get_request_ip::get_request_ip; use log::{error, info, warn}; use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; @@ -24,11 +25,10 @@ use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; -use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{InsertBatchArgs, PromptContextItem, insert_batch_prompt_context_items}; use sqlx::Acquire; use utoipa::ToSchema; -use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; /// Gpt Image 1 Image Editing #[utoipa::path( diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs index 30ca60aae0..ccf62e8077 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs @@ -1,2 +1,5 @@ pub mod flux_pro_kontext_max_edit_image_handler; -pub mod gpt_image_1_edit_image_handler; \ No newline at end of file +pub mod gemini_25_flash_edit_image_handler; +pub mod gpt_image_1_edit_image_handler; +pub mod qwen_edit_image_handler; +pub mod seededit_3_edit_image_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/qwen_edit_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/qwen_edit_image_handler.rs new file mode 100644 index 0000000000..2293d7186d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/qwen_edit_image_handler.rs @@ -0,0 +1,267 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::edit::qwen_edit_image::{QwenEditImageAcceleration, QwenEditImageNumImages, QwenEditImageRequest, QwenEditImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_qwen_image_edit_webhook::{enqueue_qwen_image_edit_webhook, QwenImageEditArgs, QwenImageEditNumImages}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Qwen Image Editing +#[utoipa::path( + post, + tag = "Generate Images (Edit)", + path = "/v1/generate/image/edit/qwen", + responses( + (status = 200, description = "Success", body = QwenEditImageResponse), + ), + params( + ("request" = QwenEditImageRequest, description = "Payload for Request"), + ) +)] +pub async fn qwen_edit_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + // TODO: Maybe don't use the batch endpoint. + let media_tokens = vec![ + request.image_media_token.clone(), + ]; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let image_media_file = match media_files.first() { + Some(file) => file, + None => { + warn!("No media files found for tokens: {:?}", media_tokens); + return Err(CommonWebError::BadInputWithSimpleMessage( + "No media files found for provided tokens".to_string())); + } + }; + + let media_domain = get_media_domain(&http_request); + + let image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &image_media_file.public_bucket_directory_hash, + image_media_file.maybe_public_bucket_prefix.as_deref(), + image_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let num_images = match request.num_images { + Some(QwenEditImageNumImages::One) => QwenImageEditNumImages::One, + Some(QwenEditImageNumImages::Two) => QwenImageEditNumImages::Two, + Some(QwenEditImageNumImages::Three) => QwenImageEditNumImages::Three, + Some(QwenEditImageNumImages::Four) => QwenImageEditNumImages::Four, + None => QwenImageEditNumImages::One, // Default to One + }; + + let acceleration = request.acceleration.as_ref().map(|accel| match accel { + QwenEditImageAcceleration::None => "none", + QwenEditImageAcceleration::Regular => "regular", + QwenEditImageAcceleration::High => "high", + }); + + let args = QwenImageEditArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + image_url: image_url, + num_images: Some(num_images), + image_size: None, + negative_prompt: request.negative_prompt.clone(), + acceleration: acceleration.map(|s| s.to_string()), + }; + + let fal_result = enqueue_qwen_image_edit_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_qwen_image_edit_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Qwen), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: vec![ + PromptContextItem { + media_token: image_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgsrc, + } + ], + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(QwenEditImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/seededit_3_edit_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/seededit_3_edit_image_handler.rs new file mode 100644 index 0000000000..7c563c20fb --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/seededit_3_edit_image_handler.rs @@ -0,0 +1,249 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::edit::seededit_3_edit_image::{SeedEdit3EditImageRequest, SeedEdit3EditImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_seededit_v3_edit_webhook::{enqueue_seededit_v3_edit_webhook, SeedEditV3EditArgs}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// SeedEdit v3 Image Editing +#[utoipa::path( + post, + tag = "Generate Images (Edit)", + path = "/v1/generate/image/edit/SeedEdit3", + responses( + (status = 200, description = "Success", body = SeedEdit3EditImageResponse), + ), + params( + ("request" = SeedEdit3EditImageRequest, description = "Payload for Request"), + ) +)] +pub async fn seededit_3_edit_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + // TODO: Maybe don't use the batch endpoint. + let media_tokens = vec![ + request.image_media_token.clone(), + ]; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let image_media_file = match media_files.first() { + Some(file) => file, + None => { + warn!("No media files found for tokens: {:?}", media_tokens); + return Err(CommonWebError::BadInputWithSimpleMessage( + "No media files found for provided tokens".to_string())); + } + }; + + let media_domain = get_media_domain(&http_request); + + let image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &image_media_file.public_bucket_directory_hash, + image_media_file.maybe_public_bucket_prefix.as_deref(), + image_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let args = SeedEditV3EditArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + image_url: image_url, + }; + + let fal_result = enqueue_seededit_v3_edit_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_seededit_v3_image_edit_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::SeedEdit3), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: vec![ + PromptContextItem { + media_token: image_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgsrc, + } + ], + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(SeedEdit3EditImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs index e078905bde..17478987cb 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs @@ -15,7 +15,6 @@ use enums::common::generation_provider::GenerationProvider; use enums::common::model_type::ModelType; use enums::common::visibility::Visibility; use fal_client::creds::open_ai_api_key::OpenAiApiKey; -use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook; use fal_client::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages}; use http_server_common::request::get_request_ip::get_request_ip; use log::{error, info, warn}; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs new file mode 100644 index 0000000000..d71887bf1e --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs @@ -0,0 +1,247 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::fetch_all_required_media_files::fetch_all_required_media_files; +use crate::util::traits::into_media_links_trait::IntoMediaLinks; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProDuration; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProImageToVideoRequest; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProImageToVideoResponse; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProResolution; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::enqueue_seedance_1_pro_image_to_video_webhook; +use fal_client::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::Seedance1ProDuration; +use fal_client::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::{Seedance1ProArgs, Seedance1ProResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, get_media_file_with_connection}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Seedance 1.0 Pro Image to Video +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/seedance_1.0_pro_image_to_video", + responses( + (status = 200, description = "Success", body = GenerateSeedance10ProImageToVideoResponse), + ), + params( + ("request" = GenerateSeedance10ProImageToVideoRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_seedance_1_0_pro_image_to_video_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(RemoveImageBackgroundError::NotAuthorized); + // } + //}; + + let start_frame_media_file_token = match &request.media_file_token { + Some(token) => token, + None => { + warn!("No media file token provided"); + return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); + } + }; + + let mut tokens = vec![ + start_frame_media_file_token.clone(), + ]; + + let media_files = fetch_all_required_media_files( + &mut mysql_connection, + &tokens, + ).await?; + + for media_file in media_files.iter() { + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + } + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) + })?; + + let media_domain = get_media_domain(&http_request); + + let start_frame_url = media_files.iter() + .find(|file| &file.token == start_frame_media_file_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("Start frame media file not found after fetch"); + CommonWebError::NotFound + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let prompt = request.prompt + .as_deref() + .map(|prompt| prompt.trim()) + .unwrap_or_else(|| ""); + + let resolution = match &request.resolution { + Some(GenerateSeedance10ProResolution::FourEightyP) => Seedance1ProResolution::FourEightyP, + Some(GenerateSeedance10ProResolution::SevenTwentyP) => Seedance1ProResolution::SevenTwentyP, + Some(GenerateSeedance10ProResolution::TenEightyP) => Seedance1ProResolution::TenEightyP, + None => Seedance1ProResolution::SevenTwentyP, + }; + + let duration = match &request.duration { + Some(GenerateSeedance10ProDuration::ThreeSeconds) => Seedance1ProDuration::ThreeSeconds, + Some(GenerateSeedance10ProDuration::FourSeconds) => Seedance1ProDuration::FourSeconds, + Some(GenerateSeedance10ProDuration::FiveSeconds) => Seedance1ProDuration::FiveSeconds, + Some(GenerateSeedance10ProDuration::SixSeconds) => Seedance1ProDuration::SixSeconds, + Some(GenerateSeedance10ProDuration::SevenSeconds) => Seedance1ProDuration::SevenSeconds, + Some(GenerateSeedance10ProDuration::EightSeconds) => Seedance1ProDuration::EightSeconds, + Some(GenerateSeedance10ProDuration::NineSeconds) => Seedance1ProDuration::NineSeconds, + Some(GenerateSeedance10ProDuration::TenSeconds) => Seedance1ProDuration::TenSeconds, + Some(GenerateSeedance10ProDuration::ElevenSeconds) => Seedance1ProDuration::ElevenSeconds, + Some(GenerateSeedance10ProDuration::TwelveSeconds) => Seedance1ProDuration::TwelveSeconds, + None => Seedance1ProDuration::FiveSeconds, + }; + + let args = Seedance1ProArgs { + image_url: start_frame_url, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + duration, + resolution, + prompt, + camera_fixed: false, // TODO: Parameterize + seed: None, // TODO: Parameterize + }; + + let fal_result = enqueue_seedance_1_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_seedance_1_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Seedance10Pro), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: Some(prompt), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(GenerateSeedance10ProImageToVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs new file mode 100644 index 0000000000..4854c2b9f4 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs @@ -0,0 +1,246 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::analytics::log_browser_session_handler::LogBrowserSessionError; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use anyhow::anyhow; +use artcraft_api_defs::generate::video::generate_veo_3_fast_image_to_video::{GenerateVeo3FastDuration, GenerateVeo3FastImageToVideoRequest, GenerateVeo3FastImageToVideoResponse, GenerateVeo3FastResolution}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::video::enqueue_veo_3_fast_image_to_video_webhook::{enqueue_veo_3_fast_image_to_video_webhook, Veo3FastArgs, Veo3FastDuration, Veo3FastResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::get_media_file_with_connection; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use tokens::tokens::prompts::PromptToken; +use utoipa::ToSchema; + +/// Veo 3 Fast Image to Video +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/veo_3_fast_image_to_video", + responses( + (status = 200, description = "Success", body = GenerateVeo3FastImageToVideoResponse), + ), + params( + ("request" = GenerateVeo3FastImageToVideoRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_veo_3_fast_image_to_video_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(RemoveImageBackgroundError::NotAuthorized); + // } + //}; + + let media_file_token = match &request.media_file_token { + Some(token) => token, + None => { + warn!("No media file token provided"); + return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); + } + }; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) + })?; + const IS_MOD : bool = false; + + let media_file_lookup_result = get_media_file_with_connection( + media_file_token, + IS_MOD, + &mut mysql_connection, + ).await; + + let media_file = match media_file_lookup_result { + Ok(Some(media_file)) => media_file, + Ok(None) => { + warn!("MediaFile not found: {:?}", media_file_token); + return Err(CommonWebError::NotFound); + }, + Err(err) => { + warn!("Error looking up media_file: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + + let media_domain = get_media_domain(&http_request); + + let bucket_path = MediaFileBucketPath::from_object_hash( + &media_file.public_bucket_directory_hash, + media_file.maybe_public_bucket_prefix.as_deref(), + media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &bucket_path); + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let prompt = request.prompt + .as_deref() + .map(|prompt| prompt.trim()) + .unwrap_or_else(|| ""); + + let resolution = match &request.resolution { + Some(GenerateVeo3FastResolution::SevenTwentyP) => Veo3FastResolution::SevenTwentyP, + Some(GenerateVeo3FastResolution::TenEightyP) => Veo3FastResolution::TenEightyP, + None => Veo3FastResolution::Default, + }; + + let duration = match &request.duration { + Some(GenerateVeo3FastDuration::EightSeconds) => Veo3FastDuration::EightSeconds, + None => Veo3FastDuration::EightSeconds, + }; + + let generate_audio = request.generate_audio + .unwrap_or(false); + + let args = Veo3FastArgs { + image_url: media_links.cdn_url, + prompt, + duration, + resolution, + generate_audio, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_veo_3_fast_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_veo_3_fast_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Veo3Fast), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: Some(prompt), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(GenerateVeo3FastImageToVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_image_to_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_image_to_video_handler.rs new file mode 100644 index 0000000000..5b5bffc24c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_image_to_video_handler.rs @@ -0,0 +1,254 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::analytics::log_browser_session_handler::LogBrowserSessionError; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use anyhow::anyhow; +use artcraft_api_defs::generate::video::generate_veo_3_image_to_video::{GenerateVeo3AspectRatio, GenerateVeo3Duration, GenerateVeo3ImageToVideoRequest, GenerateVeo3ImageToVideoResponse, GenerateVeo3Resolution}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::video::enqueue_veo_3_image_to_video_webhook::{enqueue_veo_3_image_to_video_webhook, Veo3Args, Veo3AspectRatio, Veo3Duration, Veo3Resolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::get_media_file_with_connection; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use tokens::tokens::prompts::PromptToken; +use utoipa::ToSchema; + +/// Veo 3 Image to Video +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/veo_3_image_to_video", + responses( + (status = 200, description = "Success", body = GenerateVeo3ImageToVideoResponse), + ), + params( + ("request" = GenerateVeo3ImageToVideoRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_veo_3_image_to_video_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(RemoveImageBackgroundError::NotAuthorized); + // } + //}; + + let media_file_token = match &request.media_file_token { + Some(token) => token, + None => { + warn!("No media file token provided"); + return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); + } + }; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) + })?; + const IS_MOD : bool = false; + + let media_file_lookup_result = get_media_file_with_connection( + media_file_token, + IS_MOD, + &mut mysql_connection, + ).await; + + let media_file = match media_file_lookup_result { + Ok(Some(media_file)) => media_file, + Ok(None) => { + warn!("MediaFile not found: {:?}", media_file_token); + return Err(CommonWebError::NotFound); + }, + Err(err) => { + warn!("Error looking up media_file: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + + let media_domain = get_media_domain(&http_request); + + let bucket_path = MediaFileBucketPath::from_object_hash( + &media_file.public_bucket_directory_hash, + media_file.maybe_public_bucket_prefix.as_deref(), + media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &bucket_path); + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let prompt = request.prompt + .as_deref() + .map(|prompt| prompt.trim()) + .unwrap_or_else(|| ""); + + let aspect_ratio = match &request.aspect_ratio { + Some(GenerateVeo3AspectRatio::WideSixteenNine) => Veo3AspectRatio::WideSixteenNine, + Some(GenerateVeo3AspectRatio::TallNineSixteen) => Veo3AspectRatio::TallNineSixteen, + Some(GenerateVeo3AspectRatio::Square) => Veo3AspectRatio::Square, + None => Veo3AspectRatio::WideSixteenNine, + }; + + let resolution = match &request.resolution { + Some(GenerateVeo3Resolution::SevenTwentyP) => Veo3Resolution::SevenTwentyP, + Some(GenerateVeo3Resolution::TenEightyP) => Veo3Resolution::TenEightyP, + None => Veo3Resolution::Default, + }; + + let duration = match &request.duration { + Some(GenerateVeo3Duration::EightSeconds) => Veo3Duration::EightSeconds, + None => Veo3Duration::EightSeconds, + }; + + let generate_audio = request.generate_audio + .unwrap_or(false); + + let args = Veo3Args { + image_url: media_links.cdn_url, + prompt, + duration, + resolution, + aspect_ratio, + generate_audio, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_veo_3_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_veo_3_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Veo3), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: Some(prompt), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(GenerateVeo3ImageToVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index 0dacc10a6c..ad5d429cef 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -2,4 +2,7 @@ pub mod generate_kling_1_6_pro_video_handler; pub mod generate_kling_2_1_master_video_handler; pub mod generate_kling_2_1_pro_video_handler; pub mod generate_seedance_1_0_lite_image_to_video_handler; -pub mod generate_veo_2_image_to_video_handler; \ No newline at end of file +pub mod generate_seedance_1_0_pro_image_to_video_handler; +pub mod generate_veo_2_image_to_video_handler; +pub mod generate_veo_3_fast_image_to_video_handler; +pub mod generate_veo_3_image_to_video_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index 2132e2f92f..5357bb98a9 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -1,10 +1,14 @@ use crate::http_server::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image_handler::flux_pro_kontext_max_edit_image_handler; +use crate::http_server::endpoints::generate::image::edit::gemini_25_flash_edit_image_handler::gemini_25_flash_edit_image_handler; use crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler; +use crate::http_server::endpoints::generate::image::edit::qwen_edit_image_handler::qwen_edit_image_handler; +use crate::http_server::endpoints::generate::image::edit::seededit_3_edit_image_handler::seededit_3_edit_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_1_dev_text_to_image_handler::generate_flux_1_dev_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_1_schnell_text_to_image_handler::generate_flux_1_schnell_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_pro_11_text_to_image_handler::generate_flux_pro_11_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler; +use crate::http_server::endpoints::generate::image::inpaint::flux_dev_juggernaut_inpaint_handler::flux_dev_juggernaut_inpaint_image_handler; use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_handler::flux_pro_1_inpaint_image_handler; use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler; @@ -13,12 +17,14 @@ use crate::http_server::endpoints::generate::video::generate_kling_1_6_pro_video use crate::http_server::endpoints::generate::video::generate_kling_2_1_master_video_handler::generate_kling_2_1_master_video_handler; use crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler; use crate::http_server::endpoints::generate::video::generate_seedance_1_0_lite_image_to_video_handler::generate_seedance_1_0_lite_image_to_video_handler; +use crate::http_server::endpoints::generate::video::generate_seedance_1_0_pro_image_to_video_handler::generate_seedance_1_0_pro_image_to_video_handler; use crate::http_server::endpoints::generate::video::generate_veo_2_image_to_video_handler::generate_veo_2_image_to_video_handler; +use crate::http_server::endpoints::generate::video::generate_veo_3_fast_image_to_video_handler::generate_veo_3_fast_image_to_video_handler; +use crate::http_server::endpoints::generate::video::generate_veo_3_image_to_video_handler::generate_veo_3_image_to_video_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::{web, App, Error, HttpResponse}; -use crate::http_server::endpoints::generate::image::inpaint::flux_dev_juggernaut_inpaint_handler::flux_dev_juggernaut_inpaint_image_handler; pub fn add_generate_routes (app: App) -> App where @@ -38,11 +44,22 @@ where .route(web::post().to(flux_pro_kontext_max_edit_image_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/gemini_25_flash") + .route(web::post().to(gemini_25_flash_edit_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/gpt_image_1") .route(web::post().to(gpt_image_1_edit_image_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) - + .service(web::resource("/qwen") + .route(web::post().to(qwen_edit_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/seededit_3") + .route(web::post().to(seededit_3_edit_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service(web::scope("/inpaint") .service(web::resource("/flux_dev_juggernaut") @@ -96,10 +113,22 @@ where .route(web::post().to(generate_seedance_1_0_lite_image_to_video_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/seedance_1.0_pro_image_to_video") + .route(web::post().to(generate_seedance_1_0_pro_image_to_video_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/veo_2_image_to_video") .route(web::post().to(generate_veo_2_image_to_video_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/veo_3_image_to_video") + .route(web::post().to(generate_veo_3_image_to_video_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/veo_3_fast_image_to_video") + .route(web::post().to(generate_veo_3_fast_image_to_video_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service(web::scope("/object") .service(web::resource("/hunyuan_2.0_image_to_3d") diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs index 06fc738325..4f44632979 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs @@ -26,6 +26,9 @@ pub struct ImageToVideoRequest { pub prompt: String, pub seed: i32, + + #[serde(skip_serializing_if = "Option::is_none")] + pub enable_safety_checker: Option, } #[derive(Debug, Serialize, Deserialize, Default)] diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs index 6cbbc2be2a..c9b19f4f98 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/mod.rs @@ -11,4 +11,5 @@ feature = "endpoints_fal-ai_bytedance" ))) )] -pub mod lite; \ No newline at end of file +pub mod lite; +pub mod pro; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/pro/image_to_video.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/pro/image_to_video.rs new file mode 100644 index 0000000000..22d3aa8709 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/pro/image_to_video.rs @@ -0,0 +1,62 @@ +#[allow(unused_imports)] +use crate::prelude::*; +#[allow(unused_imports)] +use serde::{Deserialize, Serialize}; +#[allow(unused_imports)] +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SeedanceImageToVideoProRequest { + pub prompt: String, + + pub image_url: String, + + /// The resolution of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub resolution: Option, + + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub camera_fixed: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub enable_safety_checker: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct I2VOutput { + /// The generated video + /// {"url":"https://v2.fal.media/files/36087878b0c1435bb75c19b64b7db178_output.mp4"} + pub video: File, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The URL where the file can be downloaded from. + pub url: String, + + // /// The mime type of the file. + // /// "image/png" + // #[serde(skip_serializing_if = "Option::is_none")] + // pub content_type: Option, + // /// File data + // #[serde(skip_serializing_if = "Option::is_none")] + // pub file_data: Option, + // /// The name of the file. It will be auto-generated if not provided. + // /// "z9RV14K95DvU.png" + // #[serde(skip_serializing_if = "Option::is_none")] + // pub file_name: Option, + // /// The size of the file in bytes. + // /// 4404019 + // #[serde(skip_serializing_if = "Option::is_none")] + // pub file_size: Option, +} + +pub fn seedance_v1_pro_image_to_video( + params: SeedanceImageToVideoProRequest, +) -> FalRequest { + FalRequest::new("fal-ai/bytedance/seedance/v1/pro/image-to-video", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/pro/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/pro/mod.rs new file mode 100644 index 0000000000..e2f49a5500 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/pro/mod.rs @@ -0,0 +1 @@ +pub mod image_to_video; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs new file mode 100644 index 0000000000..0a07eaa3ee --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs @@ -0,0 +1,53 @@ +use crate::prelude::{Deserialize, FalRequest, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Gemini25FlashImageEditInput { + pub prompt: String, + + pub image_urls: Vec, + + /// 1 - 4 + #[serde(skip_serializing_if = "Option::is_none")] + pub num_images: Option, + + /// "jpeg" or "png" + #[serde(skip_serializing_if = "Option::is_none")] + pub output_format: Option, + + // /// Resolution, eg. "720p" + // #[serde(skip_serializing_if = "Option::is_none")] + // pub resolution: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The URL where the file can be downloaded from. + pub url: String, + + // /// The mime type of the file. + // /// "image/png" + // #[serde(skip_serializing_if = "Option::is_none")] + // pub content_type: Option, + // /// File data + // #[serde(skip_serializing_if = "Option::is_none")] + // pub file_data: Option, + // /// The name of the file. It will be auto-generated if not provided. + // /// "z9RV14K95DvU.png" + // #[serde(skip_serializing_if = "Option::is_none")] + // pub file_name: Option, + // /// The size of the file in bytes. + // /// 4404019 + // #[serde(skip_serializing_if = "Option::is_none")] + // pub file_size: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Gemini25FlashImageEditOutput { + pub images: Vec, +} + +pub fn gemini_25_flash_image_edit( + params: Gemini25FlashImageEditInput, +) -> FalRequest { + FalRequest::new("fal-ai/gemini-25-flash-image/edit", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs new file mode 100644 index 0000000000..0a4a4d0e41 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs @@ -0,0 +1 @@ +pub mod edit; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs index 2403b0d6c0..1374972ee0 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/mod.rs @@ -893,6 +893,20 @@ pub mod fooocus; feature = "endpoints_fal-ai_gemini-flash-edit" ))) )] +pub mod gemini_25_flash_image; +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_gemini-flash-edit" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_gemini-flash-edit" + ))) +)] pub mod gemini_flash_edit; #[cfg(any( feature = "endpoints", diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/veo3/image_to_video.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/veo3/image_to_video.rs index 8f150f6f67..e8677e44ea 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/veo3/image_to_video.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/veo3/image_to_video.rs @@ -28,26 +28,29 @@ pub struct File { #[derive(Debug, Serialize, Deserialize, Default)] pub struct ImageToVideoInput { - /// The text prompt describing how the image should be animated - /// "A lego chef cooking eggs" - pub prompt: String, - /// URL of the input image to animate. Should be 720p or higher resolution. /// "https://fal.media/files/elephant/6fq8JDSjb1osE_c3J_F2H.png" pub image_url: String, + /// The text prompt describing how the image should be animated + /// "A lego chef cooking eggs" + pub prompt: String, + /// The duration of the generated video in seconds /// eg "8s". #[serde(skip_serializing_if = "Option::is_none")] pub duration: Option, - /// Generate audio #[serde(skip_serializing_if = "Option::is_none")] - pub generate_audio: Option, + pub aspect_ratio: Option, /// Resolution, eg. "720p" #[serde(skip_serializing_if = "Option::is_none")] pub resolution: Option, + + /// Generate audio + #[serde(skip_serializing_if = "Option::is_none")] + pub generate_audio: Option, } #[derive(Debug, Serialize, Deserialize)] diff --git a/frontend/libs/model-list/src/lib/lists/ImageModels.ts b/frontend/libs/model-list/src/lib/lists/ImageModels.ts index ad48147864..571acda1c0 100644 --- a/frontend/libs/model-list/src/lib/lists/ImageModels.ts +++ b/frontend/libs/model-list/src/lib/lists/ImageModels.ts @@ -63,6 +63,20 @@ export const IMAGE_MODELS: ImageModel[] = [ maxGenerationCount: 4, defaultGenerationCount: 4, }), + new ImageModel({ + id: "gemini_25_flash", + tauriId: "gemini_25_flash", + fullName: "Gemini 2.5 Flash", + category: "image", + creator: ModelCreator.Google, + selectorName: "Gemini 2.5 Flash", + selectorDescription: "Ultra instructive model", + selectorBadges: ["35 sec."], + maxGenerationCount: 4, + defaultGenerationCount: 1, + canEditImages: true, + tags: [ModelTag.InstructiveEdit], + }), new ImageModel({ id: "gpt_image_1", tauriId: "gpt_image_1",