From d4323da8deec15f3a20c126e0d8aec3e140ba792 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 31 Jul 2025 15:46:04 -0400 Subject: [PATCH] nearly end-to-end Flux Kontext and Flux Inpainting (#750) --- ...queue_flux_pro_kontext_max_edit_webhook.rs | 89 ++++++ .../src/requests/webhook/image/edit/mod.rs | 1 + .../enqueue_flux_pro_1_infill_webhook.rs | 92 ++++++ .../src/requests/webhook/image/infill/mod.rs | 1 + .../src/requests/webhook/image/mod.rs | 2 + crates/lib/test_data/src/web/image_urls.rs | 9 + .../edit/flux_pro_kontext_max_edit_image.rs | 36 +++ .../src/generate/image/edit/mod.rs | 1 + .../image/inpaint/flux_pro_1_inpaint_image.rs | 39 +++ .../src/generate/image/inpaint/mod.rs | 1 + .../src/generate/image/mod.rs | 1 + .../prompt_context_semantic_type.rs | 22 +- .../public/enums/src/common/model_type.rs | 24 +- ...flux_pro_kontext_max_edit_image_handler.rs | 258 ++++++++++++++++ .../endpoints/generate/image/edit/mod.rs | 1 + .../inpaint/flux_pro_1_inpaint_handler.rs | 286 ++++++++++++++++++ .../endpoints/generate/image/inpaint/mod.rs | 1 + .../endpoints/generate/image/mod.rs | 1 + .../http_server/routes/add_generate_routes.rs | 13 + .../endpoints/fal_ai/flux_pro/kontext/max.rs | 65 ++++ .../fal_ai/flux_pro/kontext/max_multi.rs | 0 .../endpoints/fal_ai/flux_pro/kontext/mod.rs | 2 + .../src/endpoints/fal_ai/flux_pro/mod.rs | 16 + 23 files changed, 959 insertions(+), 2 deletions(-) create mode 100644 crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_flux_pro_kontext_max_edit_webhook.rs create mode 100644 crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs create mode 100644 crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_pro_1_infill_webhook.rs create mode 100644 crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs create mode 100644 crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/flux_pro_kontext_max_edit_image.rs create mode 100644 crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_pro_1_inpaint_image.rs create mode 100644 crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/flux_pro_kontext_max_edit_image_handler.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs create mode 100644 crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max.rs create mode 100644 crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max_multi.rs create mode 100644 crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/mod.rs diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_flux_pro_kontext_max_edit_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_flux_pro_kontext_max_edit_webhook.rs new file mode 100644 index 0000000000..bc3beb6a11 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_flux_pro_kontext_max_edit_webhook.rs @@ -0,0 +1,89 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::prelude::fal_ai::flux_pro::kontext::max::{max, FluxProKontextMaxInput}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct FluxProKontextMaxArgs<'a, U: IntoUrl, R: IntoUrl> { + // Request + pub prompt: &'a str, + pub image_url: U, + pub num_images: FluxProKontextMaxNumImages, + + // Fulfillment + pub webhook_url: R, + pub api_key: &'a FalApiKey, +} + +#[derive(Copy, Clone, Debug)] +pub enum FluxProKontextMaxNumImages { + One, // Default + Two, + Three, + Four, +} + +pub async fn enqueue_flux_pro_kontext_max_edit_webhook( + args: FluxProKontextMaxArgs<'_, U, R> +) -> Result { + + let num_images = match args.num_images { + FluxProKontextMaxNumImages::One => 1, + FluxProKontextMaxNumImages::Two => 2, + FluxProKontextMaxNumImages::Three => 3, + FluxProKontextMaxNumImages::Four => 4, + }; + + let request = FluxProKontextMaxInput { + prompt: args.prompt.to_string(), + image_url: args.image_url.as_str().to_string(), + num_images: Some(num_images), + + // Maybe expose + aspect_ratio: None, + safety_tolerance: Some("5".to_string()), // NB: 5 is most tolerant + output_format: Some("png".to_string()), // png or jpeg + seed: None, + + // Constants + sync_mode: None, // Synchronous / slow + }; + + let result = max(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::{enqueue_flux_pro_kontext_max_edit_webhook, FluxProKontextMaxArgs, FluxProKontextMaxNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use test_data::web::image_urls::TALL_MOCHI_WITH_GLASSES_IMAGE_URL; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = FluxProKontextMaxArgs { + image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL, + prompt: "turn the glasses into sunglasses, make them sleek sunglasses with black rims, square shaped", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + num_images: FluxProKontextMaxNumImages::One, + }; + + let result = enqueue_flux_pro_kontext_max_edit_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs new file mode 100644 index 0000000000..b4f18ee0fd --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/mod.rs @@ -0,0 +1 @@ +pub mod enqueue_flux_pro_kontext_max_edit_webhook; \ No newline at end of file diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_pro_1_infill_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_pro_1_infill_webhook.rs new file mode 100644 index 0000000000..b1a3759c9a --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/infill/enqueue_flux_pro_1_infill_webhook.rs @@ -0,0 +1,92 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::prelude::fal_ai::flux_pro::v1::fill::fill; +use fal::prelude::fal_ai::flux_pro::v1::fill::FluxProFillInput; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct FluxPro1InfillArgs<'a, U: IntoUrl, R: IntoUrl, L: IntoUrl> { + // Request + pub prompt: &'a str, + pub image_url: U, + pub mask_url: R, + pub num_images: FluxPro1InfillNumImages, + + // Fulfillment + pub webhook_url: L, + pub api_key: &'a FalApiKey, +} + +#[derive(Copy, Clone, Debug)] +pub enum FluxPro1InfillNumImages { + One, // Default + Two, + Three, + Four, +} + +pub async fn enqueue_flux_pro_1_infill_webhook( + args: FluxPro1InfillArgs<'_, U, R, L> +) -> Result { + + let num_images = match args.num_images { + FluxPro1InfillNumImages::One => 1, + FluxPro1InfillNumImages::Two => 2, + FluxPro1InfillNumImages::Three => 3, + FluxPro1InfillNumImages::Four => 4, + }; + + let request = FluxProFillInput { + prompt: args.prompt.to_string(), + image_url: args.image_url.as_str().to_string(), + mask_url: args.mask_url.as_str().to_string(), + num_images: Some(num_images), + + // Maybe expose + safety_tolerance: Some("5".to_string()), // NB: 5 is most tolerant + output_format: Some("png".to_string()), // png or jpeg + seed: None, + + // Constants + sync_mode: None, // Synchronous / slow + }; + + let result = fill(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use test_data::web::image_urls::{TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL, TALL_MOCHI_WITH_GLASSES_IMAGE_URL}; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = FluxPro1InfillArgs { + image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL, + mask_url: TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL, + prompt: "slick sunglasses, cool glasses, reflection in glasses lenses", + api_key: &api_key, + webhook_url: "https://example.com/webhook", + num_images: FluxPro1InfillNumImages::One, + }; + + let result = enqueue_flux_pro_1_infill_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs new file mode 100644 index 0000000000..c42db82be4 --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/infill/mod.rs @@ -0,0 +1 @@ +pub mod enqueue_flux_pro_1_infill_webhook; \ No newline at end of file diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs index f2f10beab0..cd64404830 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/mod.rs @@ -1,7 +1,9 @@ +pub mod edit; pub mod enqueue_flux_1_dev_text_to_image_webhook; pub mod enqueue_flux_1_schnell_text_to_image_webhook; pub mod enqueue_flux_pro_11_text_to_image_webhook; pub mod enqueue_flux_pro_11_ultra_text_to_image_webhook; pub mod enqueue_gpt_image_1_edit_image_webhook; pub mod enqueue_gpt_image_1_text_to_image_webhook; +pub mod infill; pub mod remove_background_rembg_webhook; diff --git a/crates/lib/test_data/src/web/image_urls.rs b/crates/lib/test_data/src/web/image_urls.rs index b48a767f72..370632946c 100644 --- a/crates/lib/test_data/src/web/image_urls.rs +++ b/crates/lib/test_data/src/web/image_urls.rs @@ -23,3 +23,12 @@ pub const WHITE_HOUSE_SUNSET_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media // media token: m_z61yg5w7dnnvhb1f9sqt8jf02y61bv pub const WIDE_FALL_MOUNTAINS_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/w/6/m/j/p/w6mjpcqkky7f2f3xkyvkcv3qeedkmejs/asset_w6mjpcqkky7f2f3xkyvkcv3qeedkmejs.jpg"; + +// media token: m_m1bz02z1kkzanxy6rb4vk1kvq9de9g +pub const JUNO_AT_LAKE_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/s/v/c/p/p/svcpp441gyb6mjtt8413kf5zxw1ehy4t/upload_svcpp441gyb6mjtt8413kf5zxw1ehy4t.jpg"; + +// media token: m_5y81rcnpnrbey55da6cc2s9z4d2f45 +pub const TALL_MOCHI_WITH_GLASSES_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/n/m/f/h/a/nmfhaz9h5wsgzfyz7nh402wxrfwa7c2y/upload_nmfhaz9h5wsgzfyz7nh402wxrfwa7c2y.jpg"; + +// media token: m_7x18h6qnyay4a42kyyyw2xctasx5k9 +pub const TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/r/e/j/b/p/rejbp61m56hqx4a498xh7j47vkjd7jnh/upload_rejbp61m56hqx4a498xh7j47vkjd7jnh.png"; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/flux_pro_kontext_max_edit_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/flux_pro_kontext_max_edit_image.rs new file mode 100644 index 0000000000..3fc97aee21 --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/flux_pro_kontext_max_edit_image.rs @@ -0,0 +1,36 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const FLUX_PRO_KONTEXT_MAX_EDIT_IMAGE_PATH: &str = "/v1/generate/image/edit/flux_pro_kontext_max"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct FluxProKontextMaxEditImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + /// The image we're editing. + pub image_media_token: MediaFileToken, + + /// Number of images to generate. Default is one. + pub num_images: Option, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum FluxProKontextMaxEditImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct FluxProKontextMaxEditImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs index 5118b06859..e3014dfd6a 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/edit/mod.rs @@ -1,2 +1,3 @@ +pub mod flux_pro_kontext_max_edit_image; pub mod gpt_image_1_edit_image; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_pro_1_inpaint_image.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_pro_1_inpaint_image.rs new file mode 100644 index 0000000000..7a6fda8ae9 --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/flux_pro_1_inpaint_image.rs @@ -0,0 +1,39 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const FLUX_PRO_1_INPAINT_PATH: &str = "/v1/generate/image/inpaint/flux_pro_1"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct FluxPro1InpaintImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + /// The image we're editing. + pub image_media_token: MediaFileToken, + + /// The mask to use against the image. + pub mask_media_token: MediaFileToken, + + /// Number of images to generate. Default is one. + pub num_images: Option, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum FluxPro1InpaintImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct FluxPro1InpaintImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs new file mode 100644 index 0000000000..e1e410d1d1 --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/inpaint/mod.rs @@ -0,0 +1 @@ +pub mod flux_pro_1_inpaint_image; \ No newline at end of file diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs index 90f2d43e51..7ca4412b6e 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/image/mod.rs @@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image; pub mod generate_flux_pro_11_text_to_image; pub mod generate_flux_pro_11_ultra_text_to_image; pub mod generate_gpt_image_1_text_to_image; +pub mod inpaint; pub mod remove_image_background; \ No newline at end of file diff --git a/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs b/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs index 88cc3f6848..769b8f2287 100644 --- a/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs +++ b/crates/schema/public/enums/src/by_table/prompt_context_items/prompt_context_semantic_type.rs @@ -19,6 +19,12 @@ pub enum PromptContextSemanticType { /// Image-to-Video ending frame VidEndFrame, + /// Source image, eg. for inpainting. + Imgsrc, + + /// Image mask, eg. for inpainting. + Imgmask, + /// Standard image reference without a semantic type (e.g. Sora/ChatGPT 4o/gpt-image-1) Imgref, @@ -38,6 +44,8 @@ impl PromptContextSemanticType { match self { Self::VidStartFrame => "vid_start_frame", Self::VidEndFrame => "vid_end_frame", + Self::Imgsrc => "imgsrc", + Self::Imgmask => "imgmask", Self::Imgref => "imgref", Self::ImgrefCharacter => "imgref_character", Self::ImgrefStyle => "imgref_style", @@ -49,6 +57,8 @@ impl PromptContextSemanticType { match value { "vid_start_frame" => Ok(Self::VidStartFrame), "vid_end_frame" => Ok(Self::VidEndFrame), + "imgsrc" => Ok(Self::Imgsrc), + "imgmask" => Ok(Self::Imgmask), "imgref" => Ok(Self::Imgref), "imgref_character" => Ok(Self::ImgrefCharacter), "imgref_style" => Ok(Self::ImgrefStyle), @@ -63,6 +73,8 @@ impl PromptContextSemanticType { BTreeSet::from([ Self::VidStartFrame, Self::VidEndFrame, + Self::Imgsrc, + Self::Imgmask, Self::Imgref, Self::ImgrefCharacter, Self::ImgrefStyle, @@ -83,6 +95,8 @@ mod tests { fn test_serialization() { assert_serialization(PromptContextSemanticType::VidStartFrame, "vid_start_frame"); assert_serialization(PromptContextSemanticType::VidEndFrame, "vid_end_frame"); + assert_serialization(PromptContextSemanticType::Imgsrc, "imgsrc"); + assert_serialization(PromptContextSemanticType::Imgmask, "imgmask"); assert_serialization(PromptContextSemanticType::Imgref, "imgref"); assert_serialization(PromptContextSemanticType::ImgrefCharacter, "imgref_character"); assert_serialization(PromptContextSemanticType::ImgrefStyle, "imgref_style"); @@ -97,6 +111,8 @@ mod tests { fn to_str() { assert_eq!(PromptContextSemanticType::VidStartFrame.to_str(), "vid_start_frame"); assert_eq!(PromptContextSemanticType::VidEndFrame.to_str(), "vid_end_frame"); + assert_eq!(PromptContextSemanticType::Imgsrc.to_str(), "imgsrc"); + assert_eq!(PromptContextSemanticType::Imgmask.to_str(), "imgmask"); assert_eq!(PromptContextSemanticType::Imgref.to_str(), "imgref"); assert_eq!(PromptContextSemanticType::ImgrefCharacter.to_str(), "imgref_character"); assert_eq!(PromptContextSemanticType::ImgrefStyle.to_str(), "imgref_style"); @@ -107,6 +123,8 @@ mod tests { fn from_str() { assert_eq!(PromptContextSemanticType::from_str("vid_start_frame").unwrap(), PromptContextSemanticType::VidStartFrame); assert_eq!(PromptContextSemanticType::from_str("vid_end_frame").unwrap(), PromptContextSemanticType::VidEndFrame); + assert_eq!(PromptContextSemanticType::from_str("imgsrc").unwrap(), PromptContextSemanticType::Imgsrc); + assert_eq!(PromptContextSemanticType::from_str("imgmask").unwrap(), PromptContextSemanticType::Imgmask); assert_eq!(PromptContextSemanticType::from_str("imgref").unwrap(), PromptContextSemanticType::Imgref); assert_eq!(PromptContextSemanticType::from_str("imgref_character").unwrap(), PromptContextSemanticType::ImgrefCharacter); assert_eq!(PromptContextSemanticType::from_str("imgref_style").unwrap(), PromptContextSemanticType::ImgrefStyle); @@ -121,9 +139,11 @@ mod tests { #[test] fn all_variants() { let mut variants = PromptContextSemanticType::all_variants(); - assert_eq!(variants.len(), 6); + assert_eq!(variants.len(), 8); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidStartFrame)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidEndFrame)); + assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgsrc)); + assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgmask)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgref)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::ImgrefCharacter)); assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::ImgrefStyle)); diff --git a/crates/schema/public/enums/src/common/model_type.rs b/crates/schema/public/enums/src/common/model_type.rs index 6210791959..b134bd03b5 100644 --- a/crates/schema/public/enums/src/common/model_type.rs +++ b/crates/schema/public/enums/src/common/model_type.rs @@ -17,15 +17,23 @@ pub enum ModelType { Flux1Dev, #[serde(rename = "flux_1_schnell")] Flux1Schnell, + #[serde(rename = "flux_pro_1")] + FluxPro1, #[serde(rename = "flux_pro_1p1")] FluxPro11, #[serde(rename = "flux_pro_1p1_ultra")] FluxPro11Ultra, + #[serde(rename = "flux_pro_kontext_max")] + FluxProKontextMax, #[serde(rename = "gpt_image_1")] GptImage1, #[serde(rename = "recraft_3")] Recraft3, + //// Image Infill models + //#[serde(rename = "flux_pro_1_infill")] + //FluxPro1Infill, + // Video models #[serde(rename = "kling_1p6_pro")] Kling16Pro, @@ -57,8 +65,10 @@ impl ModelType { // Image models Self::Flux1Dev => "flux_1_dev", Self::Flux1Schnell => "flux_1_schnell", + Self::FluxPro1 => "flux_pro_1", Self::FluxPro11 => "flux_pro_1p1", Self::FluxPro11Ultra => "flux_pro_1p1_ultra", + Self::FluxProKontextMax => "flux_pro_kontext_max", Self::GptImage1 => "gpt_image_1", Self::Recraft3 => "recraft_3", @@ -80,8 +90,10 @@ impl ModelType { // Image models "flux_1_dev" => Ok(Self::Flux1Dev), "flux_1_schnell" => Ok(Self::Flux1Schnell), + "flux_pro_1" => Ok(Self::FluxPro1), "flux_pro_1p1" => Ok(Self::FluxPro11), "flux_pro_1p1_ultra" => Ok(Self::FluxPro11Ultra), + "flux_pro_kontext_max" => Ok(Self::FluxProKontextMax), "gpt_image_1" => Ok(Self::GptImage1), "recraft_3" => Ok(Self::Recraft3), @@ -107,8 +119,10 @@ impl ModelType { // Image models Self::Flux1Dev, Self::Flux1Schnell, + Self::FluxPro1, Self::FluxPro11, Self::FluxPro11Ultra, + Self::FluxProKontextMax, Self::GptImage1, Self::Recraft3, @@ -139,8 +153,10 @@ mod tests { // Image models assert_serialization(ModelType::Flux1Dev, "flux_1_dev"); assert_serialization(ModelType::Flux1Schnell, "flux_1_schnell"); + assert_serialization(ModelType::FluxPro1, "flux_pro_1"); assert_serialization(ModelType::FluxPro11, "flux_pro_1p1"); assert_serialization(ModelType::FluxPro11Ultra, "flux_pro_1p1_ultra"); + assert_serialization(ModelType::FluxProKontextMax, "flux_pro_kontext_max"); assert_serialization(ModelType::GptImage1, "gpt_image_1"); assert_serialization(ModelType::Recraft3, "recraft_3"); // Video models @@ -159,8 +175,10 @@ mod tests { // Image models assert_eq!(ModelType::Flux1Dev.to_str(), "flux_1_dev"); assert_eq!(ModelType::Flux1Schnell.to_str(), "flux_1_schnell"); + assert_eq!(ModelType::FluxPro1.to_str(), "flux_pro_1"); assert_eq!(ModelType::FluxPro11.to_str(), "flux_pro_1p1"); assert_eq!(ModelType::FluxPro11Ultra.to_str(), "flux_pro_1p1_ultra"); + assert_eq!(ModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max"); assert_eq!(ModelType::GptImage1.to_str(), "gpt_image_1"); assert_eq!(ModelType::Recraft3.to_str(), "recraft_3"); @@ -181,8 +199,10 @@ mod tests { // Image models assert_eq!(ModelType::from_str("flux_1_dev").unwrap(), ModelType::Flux1Dev); assert_eq!(ModelType::from_str("flux_1_schnell").unwrap(), ModelType::Flux1Schnell); + assert_eq!(ModelType::from_str("flux_pro_1").unwrap(), ModelType::FluxPro1); assert_eq!(ModelType::from_str("flux_pro_1p1").unwrap(), ModelType::FluxPro11); assert_eq!(ModelType::from_str("flux_pro_1p1_ultra").unwrap(), ModelType::FluxPro11Ultra); + assert_eq!(ModelType::from_str("flux_pro_kontext_max").unwrap(), ModelType::FluxProKontextMax); assert_eq!(ModelType::from_str("gpt_image_1").unwrap(), ModelType::GptImage1); assert_eq!(ModelType::from_str("recraft_3").unwrap(), ModelType::Recraft3); // Video models @@ -199,12 +219,14 @@ mod tests { #[test] fn all_variants() { let mut variants = ModelType::all_variants(); - assert_eq!(variants.len(), 13); + assert_eq!(variants.len(), 15); // Image models assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell)); + assert_eq!(variants.pop_first(), Some(ModelType::FluxPro1)); assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11)); assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11Ultra)); + assert_eq!(variants.pop_first(), Some(ModelType::FluxProKontextMax)); assert_eq!(variants.pop_first(), Some(ModelType::GptImage1)); assert_eq!(variants.pop_first(), Some(ModelType::Recraft3)); // Video models diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/flux_pro_kontext_max_edit_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/flux_pro_kontext_max_edit_image_handler.rs new file mode 100644 index 0000000000..cfd6cff002 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/flux_pro_kontext_max_edit_image_handler.rs @@ -0,0 +1,258 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest, FluxProKontextMaxEditImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::{enqueue_flux_pro_kontext_max_edit_webhook, FluxProKontextMaxArgs, FluxProKontextMaxNumImages}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Gpt Image 1 Image Editing +#[utoipa::path( + post, + tag = "Generate Images (Edit)", + path = "/v1/generate/image/edit/flux_pro_kontext_max", + responses( + (status = 200, description = "Success", body = FluxProKontextMaxEditImageResponse), + ), + params( + ("request" = FluxProKontextMaxEditImageRequest, description = "Payload for Request"), + ) +)] +pub async fn flux_pro_kontext_max_edit_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + // TODO: Maybe don't use the batch endpoint. + let media_tokens = vec![ + request.image_media_token.clone(), + ]; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let image_media_file = match media_files.first() { + Some(file) => file, + None => { + warn!("No media files found for tokens: {:?}", media_tokens); + return Err(CommonWebError::BadInputWithSimpleMessage( + "No media files found for provided tokens".to_string())); + } + }; + + let media_domain = get_media_domain(&http_request); + + let image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &image_media_file.public_bucket_directory_hash, + image_media_file.maybe_public_bucket_prefix.as_deref(), + image_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let num_images = match request.num_images { + Some(FluxProKontextMaxEditImageNumImages::One) => FluxProKontextMaxNumImages::One, + Some(FluxProKontextMaxEditImageNumImages::Two) => FluxProKontextMaxNumImages::Two, + Some(FluxProKontextMaxEditImageNumImages::Three) => FluxProKontextMaxNumImages::Three, + Some(FluxProKontextMaxEditImageNumImages::Four) => FluxProKontextMaxNumImages::Four, + None => FluxProKontextMaxNumImages::One, // Default to One + }; + + let args = FluxProKontextMaxArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + image_url: image_url, + num_images, + }; + + let fal_result = enqueue_flux_pro_kontext_max_edit_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_flux_pro_kontext_max_edit_webook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::FluxProKontextMax), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: vec![ + PromptContextItem { + media_token: image_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgsrc, + } + ], + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(FluxProKontextMaxEditImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs index 9cc8daa15b..30ca60aae0 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/mod.rs @@ -1 +1,2 @@ +pub mod flux_pro_kontext_max_edit_image_handler; pub mod gpt_image_1_edit_image_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs new file mode 100644 index 0000000000..e078905bde --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/flux_pro_1_inpaint_handler.rs @@ -0,0 +1,286 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest, FluxPro1InpaintImageResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook; +use fal_client::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Flux Pro-1 Inpainting +#[utoipa::path( + post, + tag = "Generate Images (Inpaint)", + path = "/v1/generate/image/inpaint/flux_pro_1", + responses( + (status = 200, description = "Success", body = FluxPro1InpaintImageResponse), + ), + params( + ("request" = FluxPro1InpaintImageRequest, description = "Payload for Request"), + ) +)] +pub async fn flux_pro_1_inpaint_image_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + const CAN_SEE_DELETED: bool = false; + + let media_tokens = vec![ + request.image_media_token.clone(), + request.mask_media_token.clone(), + ]; + + let result = batch_get_media_files_by_tokens_with_connection( + &mut mysql_connection, + &media_tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if media_files.len() != media_tokens.len() { + warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), media_tokens.len()); + return Err(CommonWebError::BadInputWithSimpleMessage( + format!("Not all media files could be found. Media files found: {}, tokens provided: {}", + media_files.len(), media_tokens.len()))); + } + + let media_domain = get_media_domain(&http_request); + + let image_media_file = media_files.iter() + .find(|file| file.token == request.image_media_token) + .ok_or_else(|| { + warn!("Image media file not found for token: {}", request.image_media_token); + CommonWebError::BadInputWithSimpleMessage("Image media file not found".to_string()) + })?; + + let mask_media_file = media_files.iter() + .find(|file| file.token == request.mask_media_token) + .ok_or_else(|| { + warn!("Mask media file not found for token: {}", request.mask_media_token); + CommonWebError::BadInputWithSimpleMessage("Mask media file not found".to_string()) + })?; + + let image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &image_media_file.public_bucket_directory_hash, + image_media_file.maybe_public_bucket_prefix.as_deref(), + image_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + let mask_image_url = { + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &mask_media_file.public_bucket_directory_hash, + mask_media_file.maybe_public_bucket_prefix.as_deref(), + mask_media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_state.server_environment, + &public_bucket_path); + + media_links.cdn_url.to_string() + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let num_images = match request.num_images { + Some(FluxPro1InpaintImageNumImages::One) => FluxPro1InfillNumImages::One, + Some(FluxPro1InpaintImageNumImages::Two) => FluxPro1InfillNumImages::Two, + Some(FluxPro1InpaintImageNumImages::Three) => FluxPro1InfillNumImages::Three, + Some(FluxPro1InpaintImageNumImages::Four) => FluxPro1InfillNumImages::Four, + None => FluxPro1InfillNumImages::One, // Default to One + }; + + let args = FluxPro1InfillArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + image_url: image_url, + mask_url: mask_image_url, + num_images, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_flux_pro_1_infill_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_flux_pro_1_infill_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + // TODO(bt,2025-07-31): Should we have an "inpaint" specific variant? + // Depends on how we want to model provider / feature matrices, routing, and results going forward. + maybe_model_type: Some(ModelType::FluxPro1), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: vec![ + PromptContextItem { + media_token: image_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgsrc, + }, + PromptContextItem { + media_token: mask_media_file.token.clone(), + context_semantic_type: PromptContextSemanticType::Imgmask, + }, + ], + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(FluxPro1InpaintImageResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs new file mode 100644 index 0000000000..f62bc9c07d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/inpaint/mod.rs @@ -0,0 +1 @@ +pub mod flux_pro_1_inpaint_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs index 378ead72b8..997a114fa5 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/mod.rs @@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image_handler; pub mod generate_flux_pro_11_text_to_image_handler; pub mod generate_flux_pro_11_ultra_text_to_image_handler; pub mod generate_gpt_image_1_text_to_image_handler; +pub mod inpaint; pub mod remove_image_background_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index 8700b54ff8..bb4f45f467 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -1,9 +1,11 @@ +use crate::http_server::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image_handler::flux_pro_kontext_max_edit_image_handler; use crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_1_dev_text_to_image_handler::generate_flux_1_dev_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_1_schnell_text_to_image_handler::generate_flux_1_schnell_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_pro_11_text_to_image_handler::generate_flux_pro_11_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler; use crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler; +use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_handler::flux_pro_1_inpaint_image_handler; use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_1_image_to_3d_handler::generate_hunyuan_2_1_image_to_3d_handler; @@ -31,10 +33,21 @@ where app.service(web::scope("/v1/generate") .service(web::scope("/image") .service(web::scope("/edit") + .service(web::resource("/flux_pro_kontext_max") + .route(web::post().to(flux_pro_kontext_max_edit_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/gpt_image_1") .route(web::post().to(gpt_image_1_edit_image_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + + ) + .service(web::scope("/inpaint") + .service(web::resource("/flux_pro_1") + .route(web::post().to(flux_pro_1_inpaint_image_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service(web::resource("/flux_1_dev_text_to_image") .route(web::post().to(generate_flux_1_dev_text_to_image_handler)) diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max.rs new file mode 100644 index 0000000000..864eda925c --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max.rs @@ -0,0 +1,65 @@ +use crate::prelude::{Deserialize, FalRequest, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct FluxProKontextMaxInput { + /// The prompt to fill the masked part of the image. + /// "A knight in shining armour holding a greatshield with \"FAL\" on it" + pub prompt: String, + + /// The image URL to generate an image from. Needs to match the dimensions of the mask. + /// "https://storage.googleapis.com/falserverless/flux-lora/example-images/knight.jpeg" + pub image_url: String, + + /// The number of images to generate. + #[serde(skip_serializing_if = "Option::is_none")] + pub num_images: Option, + + /// The format of the generated image. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_format: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, + + /// The safety tolerance level for the generated image. 1 being the most strict and 5 being the most permissive. + #[serde(skip_serializing_if = "Option::is_none")] + pub safety_tolerance: Option, + + /// The same seed and the same prompt given to the same version of the model + /// will output the same image every time. + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + + /// If set to true, the function will wait for the image to be generated and uploaded + /// before returning the response. This will increase the latency of the function but + /// it allows you to get the image directly in the response without going through the CDN. + #[serde(skip_serializing_if = "Option::is_none")] + pub sync_mode: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Image { + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + pub height: i64, + pub url: String, + pub width: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Output { + /// Whether the generated images contain NSFW concepts. + pub has_nsfw_concepts: Vec, + /// The generated image files info. + pub images: Vec, + /// The prompt used for generating the image. + pub prompt: String, + /// Seed of the generated Image. It will be the same value of the one passed in the + /// input or the randomly generated that was used in case none was passed. + pub seed: i64, + //pub timings: Timings, +} + +pub fn max(params: FluxProKontextMaxInput) -> FalRequest { + FalRequest::new("fal-ai/flux-pro/kontext/max", params) +} diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max_multi.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/max_multi.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/mod.rs new file mode 100644 index 0000000000..b1db7f8aa2 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/kontext/mod.rs @@ -0,0 +1,2 @@ +pub mod max; +pub mod max_multi; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/mod.rs index f2c11cd968..ae99e86a6a 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/flux_pro/mod.rs @@ -19,6 +19,22 @@ use std::collections::HashMap; ))) )] pub mod finetuned; + +#[cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_flux-kontext" +))] +#[cfg_attr( + docsrs, + doc(cfg(any( + feature = "endpoints", + feature = "endpoints_fal-ai", + feature = "endpoints_fal-ai_flux-kontext" + ))) +)] +pub mod kontext; + #[cfg(any( feature = "endpoints", feature = "endpoints_fal-ai",