diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs index 90cc463bbe..fbda1877df 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs @@ -4,6 +4,7 @@ use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintE use crate::core::commands::enqueue::image_inpaint::flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint; use crate::core::commands::enqueue::image_inpaint::flux_pro_1_inpaint::handle_flux_pro_1_inpaint::handle_flux_pro_1_inpaint; use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint; +use crate::core::commands::enqueue::image_inpaint::gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType}; @@ -48,16 +49,26 @@ use tokens::tokens::media_files::MediaFileToken; #[derive(Deserialize, Debug, Copy, Clone)] #[serde(rename_all = "snake_case")] pub enum ImageInpaintModel { - // Mask-based inpainting models + /* + * Mask-based inpainting models + */ + #[serde(rename = "flux_dev_juggernaut")] FluxDevJuggernaut, + #[serde(rename = "flux_pro_1")] FluxPro1, - // Non-inpainting, instructive editing models - // NB: We're supporting these to keep the app simple and not over-complicate the javascript + /* + * Non-inpainting, instructive editing models + * NB: We're supporting these to keep the app simple and not over-complicate the javascript + */ + #[serde(rename = "flux_pro_kontext_max")] FluxProKontextMax, + + #[serde(rename = "gemini_25_flash")] + Gemini25Flash, } #[derive(Deserialize, Debug)] @@ -250,7 +261,7 @@ pub async fn handle_request( ).await? } Some(ImageInpaintModel::FluxProKontextMax) => { - // Redirect to contextual image edit handler. + // TODO: Redirect to contextual image edit handler. handle_flux_pro_kontext_inpaint( request, app, @@ -264,6 +275,21 @@ pub async fn handle_request( sora_task_queue, ).await? } + Some(ImageInpaintModel::Gemini25Flash) => { + // TODO: Redirect to contextual image edit handler. + handle_gemini_25_flash_inpaint( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + fal_creds_manager, + fal_task_queue, + sora_creds_manager, + sora_task_queue, + ).await? + } }; let result = success_event diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint.rs new file mode 100644 index 0000000000..97fe41ebc3 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint.rs @@ -0,0 +1,60 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::image_inpaint::flux_pro_kontext_inpaint::handle_flux_pro_kontext_inpaint_artcraft::handle_flux_pro_kontext_inpaint_artcraft; +use crate::core::commands::enqueue::image_inpaint::gemini_25_flash_inpaint::handle_gemini_25_flash_inpaint_artcraft::handle_gemini_25_flash_inpaint_artcraft; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use log::info; +use tauri::AppHandle; + +pub async fn handle_gemini_25_flash_inpaint( + request: &EnqueueInpaintImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough (for now) + } + Provider::Sora => { + // Fallthrough + } + Provider::Artcraft => { + info!("Dispatching gemini 25 flash inpaint via Artcraft..."); + return handle_gemini_25_flash_inpaint_artcraft( + request, + app, + app_data_root, + app_env_configs, + storyteller_creds_manager + ).await; + } + } + } + + Err(InternalImageInpaintError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint_artcraft.rs new file mode 100644 index 0000000000..d9554431b5 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/handle_gemini_25_flash_inpaint_artcraft.rs @@ -0,0 +1,167 @@ +use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand; +use crate::core::commands::enqueue::image_bg_removal::errors::InternalBgRemovalError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest}; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest}; +use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest}; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use idempotency::uuid::generate_random_uuid; +use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; +use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; +use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; +use log::{error, info}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet; +use storyteller_client::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; +use storyteller_client::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image; +use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image; +use storyteller_client::generate::image::inpaint::flux_pro_1_inpaint_image::flux_pro_1_inpaint_image; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::media_files::upload_image_media_file_from_bytes::{upload_image_media_file_from_bytes, ImageType, UploadImageBytesArgs}; +use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use tauri::AppHandle; +use tokens::tokens::media_files::MediaFileToken; + +pub async fn handle_gemini_25_flash_inpaint_artcraft( + request: &EnqueueInpaintImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalImageInpaintError::NeedsStorytellerCredentials); + }, + }; + + let image_media_token = match &request.image_media_token { + Some(token) => token.clone(), + None => { + return Err(InternalImageInpaintError::NoSourceImageSpecified); + }, + }; + + info!("Calling Artcraft gemini 2.5 flash inpaint ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let num_images = match request.image_count { + None => None, + Some(1) => Some(Gemini25FlashEditImageNumImages::One), + Some(2) => Some(Gemini25FlashEditImageNumImages::Two), + Some(3) => Some(Gemini25FlashEditImageNumImages::Three), + Some(4) => Some(Gemini25FlashEditImageNumImages::Four), + Some(other) => { + return Err(InternalImageInpaintError::InvalidNumberOfRequestedImages { + min: 1, + max: 4, + requested: other, + }); + }, + }; + + let request = Gemini25FlashEditImageRequest { + uuid_idempotency_token, + prompt: Some(request.prompt.clone()), + image_media_tokens: Some(vec![image_media_token]), + num_images, + image_quality: None, + }; + + let result = gemini_25_flash_edit_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + let job_id = match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft gemini 2.5 flash inpaint. Job token: {}", + enqueued.inference_job_token); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft gemini 2.5 flash inpaint: {:?}", err); + return Err(InternalImageInpaintError::StorytellerError(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Artcraft, + model: Some(GenerationModel::Gemini25Flash), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::ImageInpaintEdit, + }) +} + +// async fn get_mask( +// request: &EnqueueInpaintImageCommand, +// app_env_configs: &AppEnvConfigs, +// storyteller_creds: &StorytellerCredentialSet, +// ) -> Result { +// +// if request.mask_image_media_token.is_some() && request.mask_image_raw_bytes.is_some() { +// return Err(InternalImageInpaintError::MaskMediaTokenAndBytesSupplied); +// } +// +// if let Some(token) = request.mask_image_media_token.as_ref() { +// return Ok(token.clone()); +// }; +// +// let image_bytes = request.mask_image_raw_bytes.as_ref() +// .ok_or(InternalImageInpaintError::NoMaskImageSpecified)?; +// +// let image_bytes = normalize_image_bytes_to_flux_mask(image_bytes) +// .map_err(|err| { +// error!("Failed to convert image bytes to png: {:?}", err); +// InternalImageInpaintError::CouldNotEncodeMask +// })?; +// +// info!("Uploading image media file from bytes..."); +// +// let result = upload_image_media_file_from_bytes(UploadImageBytesArgs { +// api_host: &app_env_configs.storyteller_host, +// maybe_creds: Some(&storyteller_creds), +// image_bytes: image_bytes.0, +// image_type: ImageType::Png, +// is_intermediate_system_file: true, +// }).await +// .map_err(|err| { +// error!("Failed to upload image media file: {:?}", err); +// InternalImageInpaintError::StorytellerError(err) +// })?; +// +// Ok(result.media_file_token) +// } diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/mod.rs new file mode 100644 index 0000000000..e5d386b210 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/gemini_25_flash_inpaint/mod.rs @@ -0,0 +1,2 @@ +pub (super) mod handle_gemini_25_flash_inpaint; +pub (super) mod handle_gemini_25_flash_inpaint_artcraft; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs index 7a0e988472..0e6e9ecf17 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs @@ -1,5 +1,6 @@ mod flux_dev_juggernaut_inpaint; mod flux_pro_1_inpaint; mod flux_pro_kontext_inpaint; +mod gemini_25_flash_inpaint; pub mod enqueue_image_inpaint_command; pub mod errors; \ No newline at end of file diff --git a/frontend/libs/model-list/src/lib/lists/ImageModels.ts b/frontend/libs/model-list/src/lib/lists/ImageModels.ts index 1acc9c56ea..571acda1c0 100644 --- a/frontend/libs/model-list/src/lib/lists/ImageModels.ts +++ b/frontend/libs/model-list/src/lib/lists/ImageModels.ts @@ -74,6 +74,7 @@ export const IMAGE_MODELS: ImageModel[] = [ selectorBadges: ["35 sec."], maxGenerationCount: 4, defaultGenerationCount: 1, + canEditImages: true, tags: [ModelTag.InstructiveEdit], }), new ImageModel({