From 9dd5e162cd9fc2f014bc57fac8ff4c2e49c5e57d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 20 Aug 2025 03:04:19 -0400 Subject: [PATCH] add flux juggernaut inpainting (just for testing - it's not SOTA) (#771) --- ...ueue_flux_dev_juggernaut_infill_webhook.rs | 96 ++++++ .../src/requests/webhook/image/infill/mod.rs | 1 + .../flux_dev_juggernaut_inpaint_image.rs | 18 ++ .../src/generate/image/inpaint/mod.rs | 3 +- .../enqueue_image_inpaint_command.rs | 17 ++ .../handle_flux_dev_juggernaut_inpaint.rs | 61 ++++ ...le_flux_dev_juggernaut_inpaint_artcraft.rs | 180 +++++++++++ .../flux_dev_juggernaut_inpaint/mod.rs | 2 + .../handle_flux_pro_1_inpaint.rs | 2 - .../commands/enqueue/image_inpaint/mod.rs | 5 +- .../commands/enqueue/task_enqueue_success.rs | 1 + .../core/events/generation_events/common.rs | 2 + .../flux_dev_juggernaut_inpaint_image.rs | 39 +++ .../src/generate/image/inpaint/mod.rs | 1 + .../public/enums/src/common/model_type.rs | 11 +- .../enums/src/tauri/tasks/task_model_type.rs | 11 +- .../flux_dev_juggernaut_inpaint_handler.rs | 285 ++++++++++++++++++ .../endpoints/generate/image/inpaint/mod.rs | 1 + .../http_server/routes/add_generate_routes.rs | 5 + .../app/src/pages/PageEdit/PageEdit.tsx | 2 +- frontend/libs/model-list/src/lib/Models.ts | 15 + .../libs/model-list/src/lib/ModelsByPage.ts | 1 + .../src/lib/enqueue/EnqueueImageInpaint.ts | 1 + 23 files changed, 752 insertions(+), 8 deletions(-) create mode 100644 crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_dev_juggernaut_infill_webhook.rs create mode 100644 crates/api_clients/storyteller_client/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/mod.rs create mode 100644 crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_dev_juggernaut_inpaint_handler.rs diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_dev_juggernaut_infill_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_dev_juggernaut_infill_webhook.rs new file mode 100644 index 000000000..e1ed4d431 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_dev_juggernaut_infill_webhook.rs @@ -0,0 +1,96 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::prelude::rundiffusion_fal::juggernaut_flux_lora::inpainting::{inpainting, InpaintInput}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct FluxDevJuggernautInfillArgs<'a, U: IntoUrl, R: IntoUrl, L: IntoUrl> { + // Request + pub prompt: &'a str, + pub image_url: U, + pub mask_url: R, + pub num_images: FluxDevJuggernautInfillNumImages, + + // Fulfillment + pub webhook_url: L, + pub api_key: &'a FalApiKey, +} + +#[derive(Copy, Clone, Debug)] +pub enum FluxDevJuggernautInfillNumImages { + One, // Default + Two, + Three, + Four, +} + +pub async fn enqueue_flux_dev_juggernaut_infill_webhook( + args: FluxDevJuggernautInfillArgs<'_, U, R, L> +) -> Result { + + let num_images = match args.num_images { + FluxDevJuggernautInfillNumImages::One => 1, + FluxDevJuggernautInfillNumImages::Two => 2, + FluxDevJuggernautInfillNumImages::Three => 3, + FluxDevJuggernautInfillNumImages::Four => 4, + }; + + let request = InpaintInput { + prompt: args.prompt.to_string(), + image_url: args.image_url.as_str().to_string(), + mask_url: args.mask_url.as_str().to_string(), + num_images: Some(num_images), + + // Maybe expose + output_format: Some("png".to_string()), // png or jpeg + seed: None, + strength: None, + loras: None, + num_inference_steps: None, + image_size: None, + enable_safety_checker: None, + guidance_scale: None, + + // Constants + sync_mode: None, // Synchronous / slow + }; + + let result = inpainting(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::infill::enqueue_flux_dev_juggernaut_infill_webhook::{enqueue_flux_dev_juggernaut_infill_webhook, FluxDevJuggernautInfillArgs, FluxDevJuggernautInfillNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use test_data::web::image_urls::{TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL, TALL_MOCHI_WITH_GLASSES_IMAGE_URL}; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = FluxDevJuggernautInfillArgs { + image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL, + mask_url: TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL, + prompt: "slick sunglasses, cool glasses, reflection in glasses lenses", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + num_images: FluxDevJuggernautInfillNumImages::Two, + }; + + let result = enqueue_flux_dev_juggernaut_infill_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs index c42db82be..9bf0945ee 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs @@ -1 +1,2 @@ +pub mod enqueue_flux_dev_juggernaut_infill_webhook; pub mod enqueue_flux_pro_1_infill_webhook; \ No newline at end of file diff --git a/crates/api_clients/storyteller_client/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs b/crates/api_clients/storyteller_client/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs new file mode 100644 index 000000000..263b10ce4 --- /dev/null +++ b/crates/api_clients/storyteller_client/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::image::inpaint::flux_dev_juggernaut_inpaint_image::{FluxDevJuggernautInpaintImageRequest, FluxDevJuggernautInpaintImageResponse, FLUX_DEV_JUGGERNAUT_INPAINT_PATH}; + +pub async fn flux_dev_juggernaut_inpaint_image( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: FluxDevJuggernautInpaintImageRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + FLUX_DEV_JUGGERNAUT_INPAINT_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/storyteller_client/src/generate/image/inpaint/mod.rs b/crates/api_clients/storyteller_client/src/generate/image/inpaint/mod.rs index e1e410d1d..32f09986f 100644 --- a/crates/api_clients/storyteller_client/src/generate/image/inpaint/mod.rs +++ b/crates/api_clients/storyteller_client/src/generate/image/inpaint/mod.rs @@ -1 +1,2 @@ -pub mod flux_pro_1_inpaint_image; \ No newline at end of file +pub mod flux_pro_1_inpaint_image; +pub mod flux_dev_juggernaut_inpaint_image; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs index d92ce410c..88b7fd262 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/enqueue_image_inpaint_command.rs @@ -44,10 +44,13 @@ use storyteller_client::media_files::get_media_file::get_media_file; use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Manager, State}; use tokens::tokens::media_files::MediaFileToken; +use crate::core::commands::enqueue::image_inpaint::flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint; #[derive(Deserialize, Debug, Copy, Clone)] #[serde(rename_all = "snake_case")] pub enum ImageInpaintModel { + #[serde(rename = "flux_dev_juggernaut")] + FluxDevJuggernaut, #[serde(rename = "flux_pro_1")] FluxPro1, } @@ -213,6 +216,20 @@ pub async fn handle_request( None => { return Err(InternalImageInpaintError::NoModelSpecified) } + Some(ImageInpaintModel::FluxDevJuggernaut) => { + handle_flux_dev_juggernaut_inpaint( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + fal_creds_manager, + fal_task_queue, + sora_creds_manager, + sora_task_queue, + ).await? + } Some(ImageInpaintModel::FluxPro1) => { handle_flux_pro_1_inpaint( request, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint.rs new file mode 100644 index 000000000..042b7acce --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint.rs @@ -0,0 +1,61 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_artcraft::handle_gpt_image_1_edit_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_edit_sora::handle_gpt_image_1_edit_sora; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::image_inpaint::flux_dev_juggernaut_inpaint::handle_flux_dev_juggernaut_inpaint_artcraft::handle_flux_dev_juggernaut_inpaint_artcraft; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use log::info; +use tauri::AppHandle; + +pub async fn handle_flux_dev_juggernaut_inpaint( + request: &EnqueueInpaintImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough (for now) + } + Provider::Sora => { + // Fallthrough + } + Provider::Artcraft => { + info!("Dispatching flux dev juggernaut inpaint via Artcraft..."); + return handle_flux_dev_juggernaut_inpaint_artcraft( + request, + app, + app_data_root, + app_env_configs, + storyteller_creds_manager + ).await; + } + } + } + + Err(InternalImageInpaintError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs new file mode 100644 index 000000000..fe6aac721 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/handle_flux_dev_juggernaut_inpaint_artcraft.rs @@ -0,0 +1,180 @@ +use crate::core::commands::enqueue::image_bg_removal::enqueue_image_bg_removal_command::EnqueueImageBgRemovalCommand; +use crate::core::commands::enqueue::image_bg_removal::errors::InternalBgRemovalError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_inpaint::errors::InternalImageInpaintError; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::model::contextual_image_edit_models::ContextualImageEditModel; +use crate::core::model::image_models::ImageModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::image::inpaint::flux_dev_juggernaut_inpaint_image::{FluxDevJuggernautInpaintImageNumImages, FluxDevJuggernautInpaintImageRequest}; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; +use idempotency::uuid::generate_random_uuid; +use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; +use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; +use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; +use log::{error, info}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image; +use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image; +use storyteller_client::generate::image::inpaint::flux_dev_juggernaut_inpaint_image::flux_dev_juggernaut_inpaint_image; +use storyteller_client::generate::image::inpaint::flux_pro_1_inpaint_image::flux_pro_1_inpaint_image; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::media_files::upload_image_media_file_from_bytes::{upload_image_media_file_from_bytes, ImageType, UploadImageBytesArgs}; +use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use tauri::AppHandle; +use tokens::tokens::media_files::MediaFileToken; + +pub async fn handle_flux_dev_juggernaut_inpaint_artcraft( + request: &EnqueueInpaintImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalImageInpaintError::NeedsStorytellerCredentials); + }, + }; + + let image_media_token = match &request.image_media_token { + Some(token) => token.clone(), + None => { + return Err(InternalImageInpaintError::NoSourceImageSpecified); + }, + }; + + //let image_media_file = get_media_file( + // &app_env_configs.storyteller_host, + // &image_media_token, + //).await.map_err(|err| { + // error!("Failed to get media file for image: {:?}", err); + // InternalImageInpaintError::StorytellerError(err) + //})?; + + let mask_media_token = get_mask( + request, + app_env_configs, + &creds, + ).await?; + + info!("Calling Artcraft flux dev juggernaut inpaint ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let num_images = match request.image_count { + None => None, + Some(1) => Some(FluxDevJuggernautInpaintImageNumImages::One), + Some(2) => Some(FluxDevJuggernautInpaintImageNumImages::Two), + Some(3) => Some(FluxDevJuggernautInpaintImageNumImages::Three), + Some(4) => Some(FluxDevJuggernautInpaintImageNumImages::Four), + Some(other) => { + return Err(InternalImageInpaintError::InvalidNumberOfRequestedImages { + min: 1, + max: 4, + requested: other, + }); + }, + }; + + let request = FluxDevJuggernautInpaintImageRequest { + uuid_idempotency_token, + prompt: Some(request.prompt.clone()), + image_media_token, + mask_media_token, + num_images, + }; + + let result = flux_dev_juggernaut_inpaint_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + let job_id = match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft flux dev juggernaut inpaint. Job token: {}", + enqueued.inference_job_token); + enqueued.inference_job_token + } + Err(err) => { + error!("Failed to use Artcraft flux dev juggernaut inpaint: {:?}", err); + return Err(InternalImageInpaintError::StorytellerError(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Artcraft, + model: Some(GenerationModel::FluxDevJuggernaut), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::ImageGeneration, + }) +} + +async fn get_mask( + request: &EnqueueInpaintImageCommand, + app_env_configs: &AppEnvConfigs, + storyteller_creds: &StorytellerCredentialSet, +) -> Result { + + if request.mask_image_media_token.is_some() && request.mask_image_raw_bytes.is_some() { + return Err(InternalImageInpaintError::MaskMediaTokenAndBytesSupplied); + } + + if let Some(token) = request.mask_image_media_token.as_ref() { + return Ok(token.clone()); + }; + + let image_bytes = request.mask_image_raw_bytes.as_ref() + .ok_or(InternalImageInpaintError::NoMaskImageSpecified)?; + + let image_bytes = normalize_image_bytes_to_flux_mask(image_bytes) + .map_err(|err| { + error!("Failed to convert image bytes to png: {:?}", err); + InternalImageInpaintError::CouldNotEncodeMask + })?; + + info!("Uploading image media file from bytes..."); + + let result = upload_image_media_file_from_bytes(UploadImageBytesArgs { + api_host: &app_env_configs.storyteller_host, + maybe_creds: Some(&storyteller_creds), + image_bytes: image_bytes.0, + image_type: ImageType::Png, + is_intermediate_system_file: true, + }).await + .map_err(|err| { + error!("Failed to upload image media file: {:?}", err); + InternalImageInpaintError::StorytellerError(err) + })?; + + Ok(result.media_file_token) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/mod.rs new file mode 100644 index 000000000..4d52ace55 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_dev_juggernaut_inpaint/mod.rs @@ -0,0 +1,2 @@ +pub (super) mod handle_flux_dev_juggernaut_inpaint; +mod handle_flux_dev_juggernaut_inpaint_artcraft; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint.rs index 3e031d76b..64ddf9fa2 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/flux_pro_1_inpaint/handle_flux_pro_1_inpaint.rs @@ -17,8 +17,6 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use log::info; use tauri::AppHandle; -pub(super) const MAX_IMAGES: usize = 10; - pub async fn handle_flux_pro_1_inpaint( request: &EnqueueInpaintImageCommand, app: &AppHandle, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs index d2721da63..ba470ccc7 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/mod.rs @@ -1,3 +1,4 @@ +mod flux_dev_juggernaut_inpaint; +mod flux_pro_1_inpaint; pub mod enqueue_image_inpaint_command; -pub mod errors; -mod flux_pro_1_inpaint; \ No newline at end of file +pub mod errors; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs b/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs index e0a789988..e78059929 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs @@ -57,6 +57,7 @@ impl TaskEnqueueSuccess{ let model_type = match self.model { None => None, Some(GenerationModel::Flux1Dev) => Some(TaskModelType::Flux1Dev), + Some(GenerationModel::FluxDevJuggernaut) => Some(TaskModelType::FluxDevJuggernaut), Some(GenerationModel::Flux1Schnell) => Some(TaskModelType::Flux1Schnell), Some(GenerationModel::FluxPro1) => Some(TaskModelType::FluxPro1), // NB: This is for inpainting. Some(GenerationModel::FluxPro11) => Some(TaskModelType::FluxPro11), diff --git a/crates/desktop/artcraft/src/core/events/generation_events/common.rs b/crates/desktop/artcraft/src/core/events/generation_events/common.rs index 548931106..6215479a8 100644 --- a/crates/desktop/artcraft/src/core/events/generation_events/common.rs +++ b/crates/desktop/artcraft/src/core/events/generation_events/common.rs @@ -16,6 +16,8 @@ pub enum GenerationModel { Flux1Dev, #[serde(rename = "flux_1_schnell")] Flux1Schnell, + #[serde(rename = "flux_dev_juggernaut")] + FluxDevJuggernaut, #[serde(rename = "flux_pro_1")] FluxPro1, #[serde(rename = "flux_pro_1.1")] diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs new file mode 100644 index 000000000..8f06022e3 --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_dev_juggernaut_inpaint_image.rs @@ -0,0 +1,39 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const FLUX_DEV_JUGGERNAUT_INPAINT_PATH: &str = "/v1/generate/image/inpaint/flux_dev_juggernaut"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct FluxDevJuggernautInpaintImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + /// The image we're editing. + pub image_media_token: MediaFileToken, + + /// The mask to use against the image. + pub mask_media_token: MediaFileToken, + + /// Number of images to generate. Default is one. + pub num_images: Option, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum FluxDevJuggernautInpaintImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct FluxDevJuggernautInpaintImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs index e1e410d1d..639ce489c 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs @@ -1 +1,2 @@ +pub mod flux_dev_juggernaut_inpaint_image; pub mod flux_pro_1_inpaint_image; \ No newline at end of file diff --git a/crates/schema/public/enums/src/common/model_type.rs b/crates/schema/public/enums/src/common/model_type.rs index 860c99ce0..5a6a186fc 100644 --- a/crates/schema/public/enums/src/common/model_type.rs +++ b/crates/schema/public/enums/src/common/model_type.rs @@ -17,6 +17,8 @@ pub enum ModelType { Flux1Dev, #[serde(rename = "flux_1_schnell")] Flux1Schnell, + #[serde(rename = "flux_dev_juggernaut")] + FluxDevJuggernaut, #[serde(rename = "flux_pro_1")] FluxPro1, #[serde(rename = "flux_pro_1p1")] @@ -83,6 +85,7 @@ impl ModelType { // Image models Self::Flux1Dev => "flux_1_dev", Self::Flux1Schnell => "flux_1_schnell", + Self::FluxDevJuggernaut => "flux_dev_juggernaut", Self::FluxPro1 => "flux_pro_1", Self::FluxPro11 => "flux_pro_1p1", Self::FluxPro11Ultra => "flux_pro_1p1_ultra", @@ -116,6 +119,7 @@ impl ModelType { // Image models "flux_1_dev" => Ok(Self::Flux1Dev), "flux_1_schnell" => Ok(Self::Flux1Schnell), + "flux_dev_juggernaut" => Ok(Self::FluxDevJuggernaut), "flux_pro_1" => Ok(Self::FluxPro1), "flux_pro_1p1" => Ok(Self::FluxPro11), "flux_pro_1p1_ultra" => Ok(Self::FluxPro11Ultra), @@ -153,6 +157,7 @@ impl ModelType { // Image models Self::Flux1Dev, Self::Flux1Schnell, + Self::FluxDevJuggernaut, Self::FluxPro1, Self::FluxPro11, Self::FluxPro11Ultra, @@ -195,6 +200,7 @@ mod tests { // Image models assert_serialization(ModelType::Flux1Dev, "flux_1_dev"); assert_serialization(ModelType::Flux1Schnell, "flux_1_schnell"); + assert_serialization(ModelType::FluxDevJuggernaut, "flux_dev_juggernaut"); assert_serialization(ModelType::FluxPro1, "flux_pro_1"); assert_serialization(ModelType::FluxPro11, "flux_pro_1p1"); assert_serialization(ModelType::FluxPro11Ultra, "flux_pro_1p1_ultra"); @@ -225,6 +231,7 @@ mod tests { // Image models assert_eq!(ModelType::Flux1Dev.to_str(), "flux_1_dev"); assert_eq!(ModelType::Flux1Schnell.to_str(), "flux_1_schnell"); + assert_eq!(ModelType::FluxDevJuggernaut.to_str(), "flux_dev_juggernaut"); assert_eq!(ModelType::FluxPro1.to_str(), "flux_pro_1"); assert_eq!(ModelType::FluxPro11.to_str(), "flux_pro_1p1"); assert_eq!(ModelType::FluxPro11Ultra.to_str(), "flux_pro_1p1_ultra"); @@ -257,6 +264,7 @@ mod tests { // Image models assert_eq!(ModelType::from_str("flux_1_dev").unwrap(), ModelType::Flux1Dev); assert_eq!(ModelType::from_str("flux_1_schnell").unwrap(), ModelType::Flux1Schnell); + assert_eq!(ModelType::from_str("flux_dev_juggernaut").unwrap(), ModelType::FluxDevJuggernaut); assert_eq!(ModelType::from_str("flux_pro_1").unwrap(), ModelType::FluxPro1); assert_eq!(ModelType::from_str("flux_pro_1p1").unwrap(), ModelType::FluxPro11); assert_eq!(ModelType::from_str("flux_pro_1p1_ultra").unwrap(), ModelType::FluxPro11Ultra); @@ -285,10 +293,11 @@ mod tests { #[test] fn all_variants() { let mut variants = ModelType::all_variants(); - assert_eq!(variants.len(), 23); + assert_eq!(variants.len(), 24); // Image models assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell)); + assert_eq!(variants.pop_first(), Some(ModelType::FluxDevJuggernaut)); assert_eq!(variants.pop_first(), Some(ModelType::FluxPro1)); assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11)); assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11Ultra)); diff --git a/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs b/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs index 0612481ea..7b67667f5 100644 --- a/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs +++ b/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs @@ -15,6 +15,8 @@ pub enum TaskModelType { Flux1Dev, #[serde(rename = "flux_1_schnell")] Flux1Schnell, + #[serde(rename = "flux_dev_juggernaut")] + FluxDevJuggernaut, // NB: For inpainting for now #[serde(rename = "flux_pro_1")] FluxPro1, @@ -64,6 +66,7 @@ impl TaskModelType { // Image models Self::Flux1Dev => "flux_1_dev", Self::Flux1Schnell => "flux_1_schnell", + Self::FluxDevJuggernaut => "flux_dev_juggernaut", Self::FluxPro1 => "flux_pro_1", Self::FluxPro11 => "flux_pro_1.1", Self::FluxPro11Ultra => "flux_pro_1.1_ultra", @@ -88,6 +91,7 @@ impl TaskModelType { // Image models "flux_1_dev" => Ok(Self::Flux1Dev), "flux_1_schnell" => Ok(Self::Flux1Schnell), + "flux_dev_juggernaut" => Ok(Self::FluxDevJuggernaut), "flux_pro_1" => Ok(Self::FluxPro1), "flux_pro_1.1" => Ok(Self::FluxPro11), "flux_pro_1.1_ultra" => Ok(Self::FluxPro11Ultra), @@ -115,6 +119,7 @@ impl TaskModelType { // Image models Self::Flux1Dev, Self::Flux1Schnell, + Self::FluxDevJuggernaut, Self::FluxPro1, Self::FluxPro11, Self::FluxPro11Ultra, @@ -148,6 +153,7 @@ mod tests { // Image models assert_serialization(TaskModelType::Flux1Dev, "flux_1_dev"); assert_serialization(TaskModelType::Flux1Schnell, "flux_1_schnell"); + assert_serialization(TaskModelType::FluxDevJuggernaut, "flux_dev_juggernaut"); assert_serialization(TaskModelType::FluxPro1, "flux_pro_1"); assert_serialization(TaskModelType::FluxPro11, "flux_pro_1.1"); assert_serialization(TaskModelType::FluxPro11Ultra, "flux_pro_1.1_ultra"); @@ -171,6 +177,7 @@ mod tests { // Image models assert_eq!(TaskModelType::Flux1Dev.to_str(), "flux_1_dev"); assert_eq!(TaskModelType::Flux1Schnell.to_str(), "flux_1_schnell"); + assert_eq!(TaskModelType::FluxDevJuggernaut.to_str(), "flux_dev_juggernaut"); assert_eq!(TaskModelType::FluxPro1.to_str(), "flux_pro_1"); assert_eq!(TaskModelType::FluxPro11.to_str(), "flux_pro_1.1"); assert_eq!(TaskModelType::FluxPro11Ultra.to_str(), "flux_pro_1.1_ultra"); @@ -194,6 +201,7 @@ mod tests { // Image models assert_eq!(TaskModelType::from_str("flux_1_dev").unwrap(), TaskModelType::Flux1Dev); assert_eq!(TaskModelType::from_str("flux_1_schnell").unwrap(), TaskModelType::Flux1Schnell); + assert_eq!(TaskModelType::from_str("flux_dev_juggernaut").unwrap(), TaskModelType::FluxDevJuggernaut); assert_eq!(TaskModelType::from_str("flux_pro_1").unwrap(), TaskModelType::FluxPro1); assert_eq!(TaskModelType::from_str("flux_pro_1.1").unwrap(), TaskModelType::FluxPro11); assert_eq!(TaskModelType::from_str("flux_pro_1.1_ultra").unwrap(), TaskModelType::FluxPro11Ultra); @@ -215,10 +223,11 @@ mod tests { #[test] fn all_variants() { let mut variants = TaskModelType::all_variants(); - assert_eq!(variants.len(), 16); + assert_eq!(variants.len(), 17); // Image models assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Schnell)); + assert_eq!(variants.pop_first(), Some(TaskModelType::FluxDevJuggernaut)); assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro1)); assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11)); assert_eq!(variants.pop_first(), Some(TaskModelType::FluxPro11Ultra)); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_dev_juggernaut_inpaint_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_dev_juggernaut_inpaint_handler.rs new file mode 100644 index 000000000..55947fb63 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_dev_juggernaut_inpaint_handler.rs @@ -0,0 +1,285 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::inpaint::flux_dev_juggernaut_inpaint_image::{FluxDevJuggernautInpaintImageNumImages, FluxDevJuggernautInpaintImageRequest, FluxDevJuggernautInpaintImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::infill::enqueue_flux_dev_juggernaut_infill_webhook::{enqueue_flux_dev_juggernaut_infill_webhook, FluxDevJuggernautInfillArgs, FluxDevJuggernautInfillNumImages}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Flux Dev juggernaut inpainting +#[utoipa::path( + post, + tag = "Generate Images (Inpaint)", + path = "/v1/generate/image/inpaint/flux_dev_juggernaut", + responses( + (status = 200, description = "Success", body = FluxDevJuggernautInpaintImageResponse), + ), + params( + ("request" = FluxDevJuggernautInpaintImageRequest, description = "Payload for Request"), + ) +)] +pub async fn flux_dev_juggernaut_inpaint_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + let media_tokens = vec![ + request.image_media_token.clone(), + request.mask_media_token.clone(), + ]; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if media_files.len() != media_tokens.len() { + warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), media_tokens.len()); + return Err(CommonWebError::BadInputWithSimpleMessage( + format!("Not all media files could be found. Media files found: {}, tokens provided: {}", + media_files.len(), media_tokens.len()))); + } + + let media_domain = get_media_domain(&http_request); + + let image_media_file = media_files.iter() + .find(|file| file.token == request.image_media_token) + .ok_or_else(|| { + warn!("Image media file not found for token: {}", request.image_media_token); + CommonWebError::BadInputWithSimpleMessage("Image media file not found".to_string()) + })?; + + let mask_media_file = media_files.iter() + .find(|file| file.token == request.mask_media_token) + .ok_or_else(|| { + warn!("Mask media file not found for token: {}", request.mask_media_token); + CommonWebError::BadInputWithSimpleMessage("Mask media file not found".to_string()) + })?; + + let image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &image_media_file.public_bucket_directory_hash, + image_media_file.maybe_public_bucket_prefix.as_deref(), + image_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + let mask_image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &mask_media_file.public_bucket_directory_hash, + mask_media_file.maybe_public_bucket_prefix.as_deref(), + mask_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let num_images = match request.num_images { + Some(FluxDevJuggernautInpaintImageNumImages::One) => FluxDevJuggernautInfillNumImages::One, + Some(FluxDevJuggernautInpaintImageNumImages::Two) => FluxDevJuggernautInfillNumImages::Two, + Some(FluxDevJuggernautInpaintImageNumImages::Three) => FluxDevJuggernautInfillNumImages::Three, + Some(FluxDevJuggernautInpaintImageNumImages::Four) => FluxDevJuggernautInfillNumImages::Four, + None => FluxDevJuggernautInfillNumImages::One, // Default to One + }; + + let args = FluxDevJuggernautInfillArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + image_url: image_url, + mask_url: mask_image_url, + num_images, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_flux_dev_juggernaut_infill_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_flux_dev_juggernaut_infill_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + // TODO(bt,2025-07-31): Should we have an "inpaint" specific variant? + // Depends on how we want to model provider / feature matrices, routing, and results going forward. + maybe_model_type: Some(ModelType::FluxDevJuggernaut), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: vec![ + PromptContextItem { + media_token: image_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgsrc, + }, + PromptContextItem { + media_token: mask_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgmask, + }, + ], + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(FluxDevJuggernautInpaintImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs index f62bc9c07..f417c22bd 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs @@ -1 +1,2 @@ +pub mod flux_dev_juggernaut_inpaint_handler; pub mod flux_pro_1_inpaint_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index bb4f45f46..2132e2f92 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -18,6 +18,7 @@ use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::{web, App, Error, HttpResponse}; +use crate::http_server::endpoints::generate::image::inpaint::flux_dev_juggernaut_inpaint_handler::flux_dev_juggernaut_inpaint_image_handler; pub fn add_generate_routes (app: App) -> App where @@ -44,6 +45,10 @@ where ) .service(web::scope("/inpaint") + .service(web::resource("/flux_dev_juggernaut") + .route(web::post().to(flux_dev_juggernaut_inpaint_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/flux_pro_1") .route(web::post().to(flux_pro_1_inpaint_image_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx b/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx index 8a8f07821..5f8eaa605 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx @@ -196,7 +196,7 @@ const PageEdit = () => { const arrayBuffer = await getMaskArrayBuffer(); await EnqueueImageInpaint({ - model: EnqueueImageInpaintModel.FluxPro1, + model: selectedModelInfo, image_media_token: editedImageToken, mask_image_raw_bytes: arrayBuffer, prompt: prompt, diff --git a/frontend/libs/model-list/src/lib/Models.ts b/frontend/libs/model-list/src/lib/Models.ts index aac23daf8..ab32322c5 100644 --- a/frontend/libs/model-list/src/lib/Models.ts +++ b/frontend/libs/model-list/src/lib/Models.ts @@ -134,6 +134,21 @@ export const ALL_MODELS: ModelConfig[] = [ }, tags: [ModelTag.MaskedInpainting], }), + cfg({ + id: "flux_dev_juggernaut_inpaint", + category: "image", + info: { + name: "Flux Dev Juggernaut (Inpainting)", + tauri_id: "flux_dev_juggernaut", + creator: mc.BlackForestLabs, + }, + description: "Fast and high-quality model", + badges: [{ label: "20 sec." }], + capabilities: { + maxGenerationCount: 4 + }, + tags: [ModelTag.MaskedInpainting], + }), ////////////////////////////// // Video models diff --git a/frontend/libs/model-list/src/lib/ModelsByPage.ts b/frontend/libs/model-list/src/lib/ModelsByPage.ts index 0650b49d4..3e997a429 100644 --- a/frontend/libs/model-list/src/lib/ModelsByPage.ts +++ b/frontend/libs/model-list/src/lib/ModelsByPage.ts @@ -31,5 +31,6 @@ export const STAGE_3D_PAGE_MODELS : ModelConfig[] = export const IMAGE_EDITOR_PAGE_MODELS : ModelConfig[] = [ // Explicit list for now. ALL_MODELS_BY_ID.get("flux_pro_inpaint")!, + ALL_MODELS_BY_ID.get("flux_dev_juggernaut_inpaint")!, ALL_MODELS_BY_ID.get("flux_pro_kontext_max")!, ]; diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts index 002059b3e..92f65839c 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueImageInpaint.ts @@ -43,6 +43,7 @@ export enum EnqueueImageInpaintErrorType { export enum EnqueueImageInpaintModel { FluxPro1 = "flux_pro_1", + FluxDevJuggernaut = "flux_dev_juggernaut", } export interface EnqueueImageInpaintError extends CommandResult {