nearly end-to-end Flux Kontext and Flux Inpainting (#750)

This commit is contained in:
Brandon Thomas authored and GitHub committed 2025-07-31 15:46:04 -04:00
1 parent 7eb2a3cdba
commit d4323da8de
23 files changed
+959 -2

No files matched your search

@@ -0,0 +1,89 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::prelude::fal_ai::flux_pro::kontext::max::{max, FluxProKontextMaxInput};
use fal::webhook::WebhookResponse;
use reqwest::IntoUrl;
pub struct FluxProKontextMaxArgs<'a, U: IntoUrl, R: IntoUrl> {
// Request
pub prompt: &'a str,
pub image_url: U,
pub num_images: FluxProKontextMaxNumImages,
// Fulfillment
pub webhook_url: R,
pub api_key: &'a FalApiKey,
}
#[derive(Copy, Clone, Debug)]
pub enum FluxProKontextMaxNumImages {
One, // Default
Two,
Three,
Four,
}
pub async fn enqueue_flux_pro_kontext_max_edit_webhook<U: IntoUrl, R: IntoUrl>(
args: FluxProKontextMaxArgs<'_, U, R>
) -> Result<WebhookResponse, FalErrorPlus> {
let num_images = match args.num_images {
FluxProKontextMaxNumImages::One => 1,
FluxProKontextMaxNumImages::Two => 2,
FluxProKontextMaxNumImages::Three => 3,
FluxProKontextMaxNumImages::Four => 4,
};
let request = FluxProKontextMaxInput {
prompt: args.prompt.to_string(),
image_url: args.image_url.as_str().to_string(),
num_images: Some(num_images),
// Maybe expose
aspect_ratio: None,
safety_tolerance: Some("5".to_string()), // NB: 5 is most tolerant
output_format: Some("png".to_string()), // png or jpeg
seed: None,
// Constants
sync_mode: None, // Synchronous / slow
};
let result = max(request)
.with_api_key(&args.api_key.0)
.queue_webhook(args.webhook_url)
.await;
result.map_err(|err| classify_fal_error(err))
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::{enqueue_flux_pro_kontext_max_edit_webhook, FluxProKontextMaxArgs, FluxProKontextMaxNumImages};
use errors::AnyhowResult;
use std::fs::read_to_string;
use test_data::web::image_urls::TALL_MOCHI_WITH_GLASSES_IMAGE_URL;
#[tokio::test]
#[ignore]
async fn test() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = FluxProKontextMaxArgs {
image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL,
prompt: "turn the glasses into sunglasses, make them sleek sunglasses with black rims, square shaped",
api_key: &api_key,
webhook_url: "https://example.com/webhook",
num_images: FluxProKontextMaxNumImages::One,
};
let result = enqueue_flux_pro_kontext_max_edit_webhook(args).await?;
Ok(())
}
}
@@ -0,0 +1 @@
pub mod enqueue_flux_pro_kontext_max_edit_webhook;
@@ -0,0 +1,92 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::prelude::fal_ai::flux_pro::v1::fill::fill;
use fal::prelude::fal_ai::flux_pro::v1::fill::FluxProFillInput;
use fal::webhook::WebhookResponse;
use reqwest::IntoUrl;
pub struct FluxPro1InfillArgs<'a, U: IntoUrl, R: IntoUrl, L: IntoUrl> {
// Request
pub prompt: &'a str,
pub image_url: U,
pub mask_url: R,
pub num_images: FluxPro1InfillNumImages,
// Fulfillment
pub webhook_url: L,
pub api_key: &'a FalApiKey,
}
#[derive(Copy, Clone, Debug)]
pub enum FluxPro1InfillNumImages {
One, // Default
Two,
Three,
Four,
}
pub async fn enqueue_flux_pro_1_infill_webhook<U: IntoUrl, R: IntoUrl, L: IntoUrl>(
args: FluxPro1InfillArgs<'_, U, R, L>
) -> Result<WebhookResponse, FalErrorPlus> {
let num_images = match args.num_images {
FluxPro1InfillNumImages::One => 1,
FluxPro1InfillNumImages::Two => 2,
FluxPro1InfillNumImages::Three => 3,
FluxPro1InfillNumImages::Four => 4,
};
let request = FluxProFillInput {
prompt: args.prompt.to_string(),
image_url: args.image_url.as_str().to_string(),
mask_url: args.mask_url.as_str().to_string(),
num_images: Some(num_images),
// Maybe expose
safety_tolerance: Some("5".to_string()), // NB: 5 is most tolerant
output_format: Some("png".to_string()), // png or jpeg
seed: None,
// Constants
sync_mode: None, // Synchronous / slow
};
let result = fill(request)
.with_api_key(&args.api_key.0)
.queue_webhook(args.webhook_url)
.await;
result.map_err(|err| classify_fal_error(err))
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages};
use errors::AnyhowResult;
use std::fs::read_to_string;
use test_data::web::image_urls::{TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL, TALL_MOCHI_WITH_GLASSES_IMAGE_URL};
#[tokio::test]
#[ignore]
async fn test() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = FluxPro1InfillArgs {
image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL,
mask_url: TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL,
prompt: "slick sunglasses, cool glasses, reflection in glasses lenses",
api_key: &api_key,
webhook_url: "https://example.com/webhook",
num_images: FluxPro1InfillNumImages::One,
};
let result = enqueue_flux_pro_1_infill_webhook(args).await?;
Ok(())
}
}
@@ -0,0 +1 @@
pub mod enqueue_flux_pro_1_infill_webhook;
@@ -1,7 +1,9 @@
pub mod edit;
pub mod enqueue_flux_1_dev_text_to_image_webhook;
pub mod enqueue_flux_1_schnell_text_to_image_webhook;
pub mod enqueue_flux_pro_11_text_to_image_webhook;
pub mod enqueue_flux_pro_11_ultra_text_to_image_webhook;
pub mod enqueue_gpt_image_1_edit_image_webhook;
pub mod enqueue_gpt_image_1_text_to_image_webhook;
pub mod infill;
pub mod remove_background_rembg_webhook;
@@ -23,3 +23,12 @@ pub const WHITE_HOUSE_SUNSET_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media
// media token: m_z61yg5w7dnnvhb1f9sqt8jf02y61bv
pub const WIDE_FALL_MOUNTAINS_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/w/6/m/j/p/w6mjpcqkky7f2f3xkyvkcv3qeedkmejs/asset_w6mjpcqkky7f2f3xkyvkcv3qeedkmejs.jpg";
// media token: m_m1bz02z1kkzanxy6rb4vk1kvq9de9g
pub const JUNO_AT_LAKE_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/s/v/c/p/p/svcpp441gyb6mjtt8413kf5zxw1ehy4t/upload_svcpp441gyb6mjtt8413kf5zxw1ehy4t.jpg";
// media token: m_5y81rcnpnrbey55da6cc2s9z4d2f45
pub const TALL_MOCHI_WITH_GLASSES_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/n/m/f/h/a/nmfhaz9h5wsgzfyz7nh402wxrfwa7c2y/upload_nmfhaz9h5wsgzfyz7nh402wxrfwa7c2y.jpg";
// media token: m_7x18h6qnyay4a42kyyyw2xctasx5k9
pub const TALL_MOCHI_WITH_GLASSES_GLASSES_MASK_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/r/e/j/b/p/rejbp61m56hqx4a498xh7j47vkjd7jnh/upload_rejbp61m56hqx4a498xh7j47vkjd7jnh.png";
@@ -0,0 +1,36 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const FLUX_PRO_KONTEXT_MAX_EDIT_IMAGE_PATH: &str = "/v1/generate/image/edit/flux_pro_kontext_max";
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxProKontextMaxEditImageRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
/// The image we're editing.
pub image_media_token: MediaFileToken,
/// Number of images to generate. Default is one.
pub num_images: Option<FluxProKontextMaxEditImageNumImages>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum FluxProKontextMaxEditImageNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxProKontextMaxEditImageResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -1,2 +1,3 @@
pub mod flux_pro_kontext_max_edit_image;
pub mod gpt_image_1_edit_image;
@@ -0,0 +1,39 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const FLUX_PRO_1_INPAINT_PATH: &str = "/v1/generate/image/inpaint/flux_pro_1";
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxPro1InpaintImageRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
/// The image we're editing.
pub image_media_token: MediaFileToken,
/// The mask to use against the image.
pub mask_media_token: MediaFileToken,
/// Number of images to generate. Default is one.
pub num_images: Option<FluxPro1InpaintImageNumImages>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum FluxPro1InpaintImageNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct FluxPro1InpaintImageResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1 @@
pub mod flux_pro_1_inpaint_image;
@@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image;
pub mod generate_flux_pro_11_text_to_image;
pub mod generate_flux_pro_11_ultra_text_to_image;
pub mod generate_gpt_image_1_text_to_image;
pub mod inpaint;
pub mod remove_image_background;
@@ -19,6 +19,12 @@ pub enum PromptContextSemanticType {
/// Image-to-Video ending frame
VidEndFrame,
/// Source image, eg. for inpainting.
Imgsrc,
/// Image mask, eg. for inpainting.
Imgmask,
/// Standard image reference without a semantic type (e.g. Sora/ChatGPT 4o/gpt-image-1)
Imgref,
@@ -38,6 +44,8 @@ impl PromptContextSemanticType {
match self {
Self::VidStartFrame => "vid_start_frame",
Self::VidEndFrame => "vid_end_frame",
Self::Imgsrc => "imgsrc",
Self::Imgmask => "imgmask",
Self::Imgref => "imgref",
Self::ImgrefCharacter => "imgref_character",
Self::ImgrefStyle => "imgref_style",
@@ -49,6 +57,8 @@ impl PromptContextSemanticType {
match value {
"vid_start_frame" => Ok(Self::VidStartFrame),
"vid_end_frame" => Ok(Self::VidEndFrame),
"imgsrc" => Ok(Self::Imgsrc),
"imgmask" => Ok(Self::Imgmask),
"imgref" => Ok(Self::Imgref),
"imgref_character" => Ok(Self::ImgrefCharacter),
"imgref_style" => Ok(Self::ImgrefStyle),
@@ -63,6 +73,8 @@ impl PromptContextSemanticType {
BTreeSet::from([
Self::VidStartFrame,
Self::VidEndFrame,
Self::Imgsrc,
Self::Imgmask,
Self::Imgref,
Self::ImgrefCharacter,
Self::ImgrefStyle,
@@ -83,6 +95,8 @@ mod tests {
fn test_serialization() {
assert_serialization(PromptContextSemanticType::VidStartFrame, "vid_start_frame");
assert_serialization(PromptContextSemanticType::VidEndFrame, "vid_end_frame");
assert_serialization(PromptContextSemanticType::Imgsrc, "imgsrc");
assert_serialization(PromptContextSemanticType::Imgmask, "imgmask");
assert_serialization(PromptContextSemanticType::Imgref, "imgref");
assert_serialization(PromptContextSemanticType::ImgrefCharacter, "imgref_character");
assert_serialization(PromptContextSemanticType::ImgrefStyle, "imgref_style");
@@ -97,6 +111,8 @@ mod tests {
fn to_str() {
assert_eq!(PromptContextSemanticType::VidStartFrame.to_str(), "vid_start_frame");
assert_eq!(PromptContextSemanticType::VidEndFrame.to_str(), "vid_end_frame");
assert_eq!(PromptContextSemanticType::Imgsrc.to_str(), "imgsrc");
assert_eq!(PromptContextSemanticType::Imgmask.to_str(), "imgmask");
assert_eq!(PromptContextSemanticType::Imgref.to_str(), "imgref");
assert_eq!(PromptContextSemanticType::ImgrefCharacter.to_str(), "imgref_character");
assert_eq!(PromptContextSemanticType::ImgrefStyle.to_str(), "imgref_style");
@@ -107,6 +123,8 @@ mod tests {
fn from_str() {
assert_eq!(PromptContextSemanticType::from_str("vid_start_frame").unwrap(), PromptContextSemanticType::VidStartFrame);
assert_eq!(PromptContextSemanticType::from_str("vid_end_frame").unwrap(), PromptContextSemanticType::VidEndFrame);
assert_eq!(PromptContextSemanticType::from_str("imgsrc").unwrap(), PromptContextSemanticType::Imgsrc);
assert_eq!(PromptContextSemanticType::from_str("imgmask").unwrap(), PromptContextSemanticType::Imgmask);
assert_eq!(PromptContextSemanticType::from_str("imgref").unwrap(), PromptContextSemanticType::Imgref);
assert_eq!(PromptContextSemanticType::from_str("imgref_character").unwrap(), PromptContextSemanticType::ImgrefCharacter);
assert_eq!(PromptContextSemanticType::from_str("imgref_style").unwrap(), PromptContextSemanticType::ImgrefStyle);
@@ -121,9 +139,11 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = PromptContextSemanticType::all_variants();
assert_eq!(variants.len(), 6);
assert_eq!(variants.len(), 8);
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidStartFrame));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::VidEndFrame));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgsrc));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgmask));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::Imgref));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::ImgrefCharacter));
assert_eq!(variants.pop_first(), Some(PromptContextSemanticType::ImgrefStyle));
@@ -17,15 +17,23 @@ pub enum ModelType {
Flux1Dev,
#[serde(rename = "flux_1_schnell")]
Flux1Schnell,
#[serde(rename = "flux_pro_1")]
FluxPro1,
#[serde(rename = "flux_pro_1p1")]
FluxPro11,
#[serde(rename = "flux_pro_1p1_ultra")]
FluxPro11Ultra,
#[serde(rename = "flux_pro_kontext_max")]
FluxProKontextMax,
#[serde(rename = "gpt_image_1")]
GptImage1,
#[serde(rename = "recraft_3")]
Recraft3,
//// Image Infill models
//#[serde(rename = "flux_pro_1_infill")]
//FluxPro1Infill,
// Video models
#[serde(rename = "kling_1p6_pro")]
Kling16Pro,
@@ -57,8 +65,10 @@ impl ModelType {
// Image models
Self::Flux1Dev => "flux_1_dev",
Self::Flux1Schnell => "flux_1_schnell",
Self::FluxPro1 => "flux_pro_1",
Self::FluxPro11 => "flux_pro_1p1",
Self::FluxPro11Ultra => "flux_pro_1p1_ultra",
Self::FluxProKontextMax => "flux_pro_kontext_max",
Self::GptImage1 => "gpt_image_1",
Self::Recraft3 => "recraft_3",
@@ -80,8 +90,10 @@ impl ModelType {
// Image models
"flux_1_dev" => Ok(Self::Flux1Dev),
"flux_1_schnell" => Ok(Self::Flux1Schnell),
"flux_pro_1" => Ok(Self::FluxPro1),
"flux_pro_1p1" => Ok(Self::FluxPro11),
"flux_pro_1p1_ultra" => Ok(Self::FluxPro11Ultra),
"flux_pro_kontext_max" => Ok(Self::FluxProKontextMax),
"gpt_image_1" => Ok(Self::GptImage1),
"recraft_3" => Ok(Self::Recraft3),
@@ -107,8 +119,10 @@ impl ModelType {
// Image models
Self::Flux1Dev,
Self::Flux1Schnell,
Self::FluxPro1,
Self::FluxPro11,
Self::FluxPro11Ultra,
Self::FluxProKontextMax,
Self::GptImage1,
Self::Recraft3,
@@ -139,8 +153,10 @@ mod tests {
// Image models
assert_serialization(ModelType::Flux1Dev, "flux_1_dev");
assert_serialization(ModelType::Flux1Schnell, "flux_1_schnell");
assert_serialization(ModelType::FluxPro1, "flux_pro_1");
assert_serialization(ModelType::FluxPro11, "flux_pro_1p1");
assert_serialization(ModelType::FluxPro11Ultra, "flux_pro_1p1_ultra");
assert_serialization(ModelType::FluxProKontextMax, "flux_pro_kontext_max");
assert_serialization(ModelType::GptImage1, "gpt_image_1");
assert_serialization(ModelType::Recraft3, "recraft_3");
// Video models
@@ -159,8 +175,10 @@ mod tests {
// Image models
assert_eq!(ModelType::Flux1Dev.to_str(), "flux_1_dev");
assert_eq!(ModelType::Flux1Schnell.to_str(), "flux_1_schnell");
assert_eq!(ModelType::FluxPro1.to_str(), "flux_pro_1");
assert_eq!(ModelType::FluxPro11.to_str(), "flux_pro_1p1");
assert_eq!(ModelType::FluxPro11Ultra.to_str(), "flux_pro_1p1_ultra");
assert_eq!(ModelType::FluxProKontextMax.to_str(), "flux_pro_kontext_max");
assert_eq!(ModelType::GptImage1.to_str(), "gpt_image_1");
assert_eq!(ModelType::Recraft3.to_str(), "recraft_3");
@@ -181,8 +199,10 @@ mod tests {
// Image models
assert_eq!(ModelType::from_str("flux_1_dev").unwrap(), ModelType::Flux1Dev);
assert_eq!(ModelType::from_str("flux_1_schnell").unwrap(), ModelType::Flux1Schnell);
assert_eq!(ModelType::from_str("flux_pro_1").unwrap(), ModelType::FluxPro1);
assert_eq!(ModelType::from_str("flux_pro_1p1").unwrap(), ModelType::FluxPro11);
assert_eq!(ModelType::from_str("flux_pro_1p1_ultra").unwrap(), ModelType::FluxPro11Ultra);
assert_eq!(ModelType::from_str("flux_pro_kontext_max").unwrap(), ModelType::FluxProKontextMax);
assert_eq!(ModelType::from_str("gpt_image_1").unwrap(), ModelType::GptImage1);
assert_eq!(ModelType::from_str("recraft_3").unwrap(), ModelType::Recraft3);
// Video models
@@ -199,12 +219,14 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = ModelType::all_variants();
assert_eq!(variants.len(), 13);
assert_eq!(variants.len(), 15);
// Image models
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev));
assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell));
assert_eq!(variants.pop_first(), Some(ModelType::FluxPro1));
assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11));
assert_eq!(variants.pop_first(), Some(ModelType::FluxPro11Ultra));
assert_eq!(variants.pop_first(), Some(ModelType::FluxProKontextMax));
assert_eq!(variants.pop_first(), Some(ModelType::GptImage1));
assert_eq!(variants.pop_first(), Some(ModelType::Recraft3));
// Video models
@@ -0,0 +1,258 @@
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{FluxProKontextMaxEditImageNumImages, FluxProKontextMaxEditImageRequest, FluxProKontextMaxEditImageResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::edit::enqueue_flux_pro_kontext_max_edit_webhook::{enqueue_flux_pro_kontext_max_edit_webhook, FluxProKontextMaxArgs, FluxProKontextMaxNumImages};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use sqlx::Acquire;
use utoipa::ToSchema;
/// Gpt Image 1 Image Editing
#[utoipa::path(
post,
tag = "Generate Images (Edit)",
path = "/v1/generate/image/edit/flux_pro_kontext_max",
responses(
(status = 200, description = "Success", body = FluxProKontextMaxEditImageResponse),
),
params(
("request" = FluxProKontextMaxEditImageRequest, description = "Payload for Request"),
)
)]
pub async fn flux_pro_kontext_max_edit_image_handler(
http_request: HttpRequest,
request: Json<FluxProKontextMaxEditImageRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<FluxProKontextMaxEditImageResponse>, CommonWebError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
const CAN_SEE_DELETED: bool = false;
// TODO: Maybe don't use the batch endpoint.
let media_tokens = vec![
request.image_media_token.clone(),
];
let result = batch_get_media_files_by_tokens_with_connection(
&mut mysql_connection,
&media_tokens,
CAN_SEE_DELETED,
).await;
let result = batch_get_media_files_by_tokens_with_connection(
&mut mysql_connection,
&media_tokens,
CAN_SEE_DELETED,
).await;
let media_files = match result {
Ok(files) => files,
Err(err) => {
error!("Error getting media files by tokens: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let image_media_file = match media_files.first() {
Some(file) => file,
None => {
warn!("No media files found for tokens: {:?}", media_tokens);
return Err(CommonWebError::BadInputWithSimpleMessage(
"No media files found for provided tokens".to_string()));
}
};
let media_domain = get_media_domain(&http_request);
let image_url = {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&image_media_file.public_bucket_directory_hash,
image_media_file.maybe_public_bucket_prefix.as_deref(),
image_media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let num_images = match request.num_images {
Some(FluxProKontextMaxEditImageNumImages::One) => FluxProKontextMaxNumImages::One,
Some(FluxProKontextMaxEditImageNumImages::Two) => FluxProKontextMaxNumImages::Two,
Some(FluxProKontextMaxEditImageNumImages::Three) => FluxProKontextMaxNumImages::Three,
Some(FluxProKontextMaxEditImageNumImages::Four) => FluxProKontextMaxNumImages::Four,
None => FluxProKontextMaxNumImages::One, // Default to One
};
let args = FluxProKontextMaxArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
image_url: image_url,
num_images,
};
let fal_result = enqueue_flux_pro_kontext_max_edit_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_flux_pro_kontext_max_edit_webook: {:?}", err);
CommonWebError::ServerError
})?;
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::FluxProKontextMax),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: vec![
PromptContextItem {
media_token: image_media_file.token.clone(),
context_semantic_type: PromptContextSemanticType::Imgsrc,
}
],
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::ImageGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(FluxProKontextMaxEditImageResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -1 +1,2 @@
pub mod flux_pro_kontext_max_edit_image_handler;
pub mod gpt_image_1_edit_image_handler;
@@ -0,0 +1,286 @@
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest, FluxPro1InpaintImageResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::enqueue_gpt_image_1_edit_image_webhook;
use fal_client::requests::webhook::image::infill::enqueue_flux_pro_1_infill_webhook::{enqueue_flux_pro_1_infill_webhook, FluxPro1InfillArgs, FluxPro1InfillNumImages};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use sqlx::Acquire;
use utoipa::ToSchema;
/// Flux Pro-1 Inpainting
#[utoipa::path(
post,
tag = "Generate Images (Inpaint)",
path = "/v1/generate/image/inpaint/flux_pro_1",
responses(
(status = 200, description = "Success", body = FluxPro1InpaintImageResponse),
),
params(
("request" = FluxPro1InpaintImageRequest, description = "Payload for Request"),
)
)]
pub async fn flux_pro_1_inpaint_image_handler(
http_request: HttpRequest,
request: Json<FluxPro1InpaintImageRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<FluxPro1InpaintImageResponse>, CommonWebError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
const CAN_SEE_DELETED: bool = false;
let media_tokens = vec![
request.image_media_token.clone(),
request.mask_media_token.clone(),
];
let result = batch_get_media_files_by_tokens_with_connection(
&mut mysql_connection,
&media_tokens,
CAN_SEE_DELETED,
).await;
let media_files = match result {
Ok(files) => files,
Err(err) => {
error!("Error getting media files by tokens: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
if media_files.len() != media_tokens.len() {
warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), media_tokens.len());
return Err(CommonWebError::BadInputWithSimpleMessage(
format!("Not all media files could be found. Media files found: {}, tokens provided: {}",
media_files.len(), media_tokens.len())));
}
let media_domain = get_media_domain(&http_request);
let image_media_file = media_files.iter()
.find(|file| file.token == request.image_media_token)
.ok_or_else(|| {
warn!("Image media file not found for token: {}", request.image_media_token);
CommonWebError::BadInputWithSimpleMessage("Image media file not found".to_string())
})?;
let mask_media_file = media_files.iter()
.find(|file| file.token == request.mask_media_token)
.ok_or_else(|| {
warn!("Mask media file not found for token: {}", request.mask_media_token);
CommonWebError::BadInputWithSimpleMessage("Mask media file not found".to_string())
})?;
let image_url = {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&image_media_file.public_bucket_directory_hash,
image_media_file.maybe_public_bucket_prefix.as_deref(),
image_media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
};
let mask_image_url = {
let public_bucket_path = MediaFileBucketPath::from_object_hash(
&mask_media_file.public_bucket_directory_hash,
mask_media_file.maybe_public_bucket_prefix.as_deref(),
mask_media_file.maybe_public_bucket_extension.as_deref());
let media_links = MediaLinksBuilder::from_media_path_and_env(
media_domain,
server_state.server_environment,
&public_bucket_path);
media_links.cdn_url.to_string()
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let num_images = match request.num_images {
Some(FluxPro1InpaintImageNumImages::One) => FluxPro1InfillNumImages::One,
Some(FluxPro1InpaintImageNumImages::Two) => FluxPro1InfillNumImages::Two,
Some(FluxPro1InpaintImageNumImages::Three) => FluxPro1InfillNumImages::Three,
Some(FluxPro1InpaintImageNumImages::Four) => FluxPro1InfillNumImages::Four,
None => FluxPro1InfillNumImages::One, // Default to One
};
let args = FluxPro1InfillArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
image_url: image_url,
mask_url: mask_image_url,
num_images,
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
let fal_result = enqueue_flux_pro_1_infill_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_flux_pro_1_infill_webhook: {:?}", err);
CommonWebError::ServerError
})?;
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
// TODO(bt,2025-07-31): Should we have an "inpaint" specific variant?
// Depends on how we want to model provider / feature matrices, routing, and results going forward.
maybe_model_type: Some(ModelType::FluxPro1),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(token) = prompt_token.as_ref() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: vec![
PromptContextItem {
media_token: image_media_file.token.clone(),
context_semantic_type: PromptContextSemanticType::Imgsrc,
},
PromptContextItem {
media_token: mask_media_file.token.clone(),
context_semantic_type: PromptContextSemanticType::Imgmask,
},
],
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::ImageGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(FluxPro1InpaintImageResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -0,0 +1 @@
pub mod flux_pro_1_inpaint_handler;
@@ -4,4 +4,5 @@ pub mod generate_flux_1_schnell_text_to_image_handler;
pub mod generate_flux_pro_11_text_to_image_handler;
pub mod generate_flux_pro_11_ultra_text_to_image_handler;
pub mod generate_gpt_image_1_text_to_image_handler;
pub mod inpaint;
pub mod remove_image_background_handler;
@@ -1,9 +1,11 @@
use crate::http_server::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image_handler::flux_pro_kontext_max_edit_image_handler;
use crate::http_server::endpoints::generate::image::edit::gpt_image_1_edit_image_handler::gpt_image_1_edit_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_1_dev_text_to_image_handler::generate_flux_1_dev_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_1_schnell_text_to_image_handler::generate_flux_1_schnell_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_pro_11_text_to_image_handler::generate_flux_pro_11_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_flux_pro_11_ultra_text_to_image_handler::generate_flux_pro_11_ultra_text_to_image_handler;
use crate::http_server::endpoints::generate::image::generate_gpt_image_1_text_to_image_handler::generate_gpt_image_1_text_to_image_handler;
use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_handler::flux_pro_1_inpaint_image_handler;
use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_1_image_to_3d_handler::generate_hunyuan_2_1_image_to_3d_handler;
@@ -31,10 +33,21 @@ where
app.service(web::scope("/v1/generate")
.service(web::scope("/image")
.service(web::scope("/edit")
.service(web::resource("/flux_pro_kontext_max")
.route(web::post().to(flux_pro_kontext_max_edit_image_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/gpt_image_1")
.route(web::post().to(gpt_image_1_edit_image_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::scope("/inpaint")
.service(web::resource("/flux_pro_1")
.route(web::post().to(flux_pro_1_inpaint_image_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)
.service(web::resource("/flux_1_dev_text_to_image")
.route(web::post().to(generate_flux_1_dev_text_to_image_handler))
@@ -0,0 +1,65 @@
use crate::prelude::{Deserialize, FalRequest, Serialize};
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct FluxProKontextMaxInput {
/// The prompt to fill the masked part of the image.
/// "A knight in shining armour holding a greatshield with \"FAL\" on it"
pub prompt: String,
/// The image URL to generate an image from. Needs to match the dimensions of the mask.
/// "https://storage.googleapis.com/falserverless/flux-lora/example-images/knight.jpeg"
pub image_url: String,
/// The number of images to generate.
#[serde(skip_serializing_if = "Option::is_none")]
pub num_images: Option<i64>,
/// The format of the generated image.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_format: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub aspect_ratio: Option<String>,
/// The safety tolerance level for the generated image. 1 being the most strict and 5 being the most permissive.
#[serde(skip_serializing_if = "Option::is_none")]
pub safety_tolerance: Option<String>,
/// The same seed and the same prompt given to the same version of the model
/// will output the same image every time.
#[serde(skip_serializing_if = "Option::is_none")]
pub seed: Option<i64>,
/// If set to true, the function will wait for the image to be generated and uploaded
/// before returning the response. This will increase the latency of the function but
/// it allows you to get the image directly in the response without going through the CDN.
#[serde(skip_serializing_if = "Option::is_none")]
pub sync_mode: Option<bool>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct Image {
#[serde(skip_serializing_if = "Option::is_none")]
pub content_type: Option<String>,
pub height: i64,
pub url: String,
pub width: i64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Output {
/// Whether the generated images contain NSFW concepts.
pub has_nsfw_concepts: Vec<bool>,
/// The generated image files info.
pub images: Vec<Image>,
/// The prompt used for generating the image.
pub prompt: String,
/// Seed of the generated Image. It will be the same value of the one passed in the
/// input or the randomly generated that was used in case none was passed.
pub seed: i64,
//pub timings: Timings,
}
pub fn max(params: FluxProKontextMaxInput) -> FalRequest<FluxProKontextMaxInput, Output> {
FalRequest::new("fal-ai/flux-pro/kontext/max", params)
}
Whitespace-only changes.
@@ -0,0 +1,2 @@
pub mod max;
pub mod max_multi;
@@ -19,6 +19,22 @@ use std::collections::HashMap;
)))
)]
pub mod finetuned;
#[cfg(any(
feature = "endpoints",
feature = "endpoints_fal-ai",
feature = "endpoints_fal-ai_flux-kontext"
))]
#[cfg_attr(
docsrs,
doc(cfg(any(
feature = "endpoints",
feature = "endpoints_fal-ai",
feature = "endpoints_fal-ai_flux-kontext"
)))
)]
pub mod kontext;
#[cfg(any(
feature = "endpoints",
feature = "endpoints_fal-ai",