diff --git a/crates/schema/public/enums/src/common/model_type.rs b/crates/schema/public/enums/src/common/model_type.rs index 90809dd27e..0c7cd6bf61 100644 --- a/crates/schema/public/enums/src/common/model_type.rs +++ b/crates/schema/public/enums/src/common/model_type.rs @@ -85,6 +85,8 @@ pub enum ModelType { Kling21Pro, #[serde(rename = "kling_2p1_master")] Kling21Master, + #[serde(rename = "kling_2p5_turbo_pro")] + Kling2p5TurboPro, #[serde(rename = "seedance_1p0_lite")] Seedance10Lite, #[serde(rename = "seedance_1p0_pro")] @@ -147,6 +149,7 @@ impl ModelType { Self::Kling16Pro => "kling_1p6_pro", Self::Kling21Pro => "kling_2p1_pro", Self::Kling21Master => "kling_2p1_master", + Self::Kling2p5TurboPro => "kling_2p5_turbo_pro", Self::Seedance10Lite => "seedance_1p0_lite", Self::Seedance10Pro => "seedance_1p0_pro", Self::Sora2 => "sora_2", @@ -195,6 +198,7 @@ impl ModelType { "kling_1p6_pro" => Ok(Self::Kling16Pro), "kling_2p1_pro" => Ok(Self::Kling21Pro), "kling_2p1_master" => Ok(Self::Kling21Master), + "kling_2p5_turbo_pro" => Ok(Self::Kling2p5TurboPro), "seedance_1p0_lite" => Ok(Self::Seedance10Lite), "seedance_1p0_pro" => Ok(Self::Seedance10Pro), "sora_2" => Ok(Self::Sora2), @@ -247,6 +251,7 @@ impl ModelType { Self::Kling16Pro, Self::Kling21Pro, Self::Kling21Master, + Self::Kling2p5TurboPro, Self::Seedance10Lite, Self::Seedance10Pro, Self::Sora2, @@ -303,6 +308,7 @@ mod tests { assert_serialization(ModelType::Kling16Pro, "kling_1p6_pro"); assert_serialization(ModelType::Kling21Pro, "kling_2p1_pro"); assert_serialization(ModelType::Kling21Master, "kling_2p1_master"); + assert_serialization(ModelType::Kling2p5TurboPro, "kling_2p5_turbo_pro"); assert_serialization(ModelType::Seedance10Lite, "seedance_1p0_lite"); assert_serialization(ModelType::Seedance10Pro, "seedance_1p0_pro"); assert_serialization(ModelType::Sora2, "sora_2"); @@ -349,6 +355,7 @@ mod tests { assert_eq!(ModelType::Kling16Pro.to_str(), "kling_1p6_pro"); assert_eq!(ModelType::Kling21Pro.to_str(), "kling_2p1_pro"); assert_eq!(ModelType::Kling21Master.to_str(), "kling_2p1_master"); + assert_eq!(ModelType::Kling2p5TurboPro.to_str(), "kling_2p5_turbo_pro"); assert_eq!(ModelType::Seedance10Lite.to_str(), "seedance_1p0_lite"); assert_eq!(ModelType::Seedance10Pro.to_str(), "seedance_1p0_pro"); assert_eq!(ModelType::Sora2.to_str(), "sora_2"); @@ -395,6 +402,7 @@ mod tests { assert_eq!(ModelType::from_str("kling_1p6_pro").unwrap(), ModelType::Kling16Pro); assert_eq!(ModelType::from_str("kling_2p1_pro").unwrap(), ModelType::Kling21Pro); assert_eq!(ModelType::from_str("kling_2p1_master").unwrap(), ModelType::Kling21Master); + assert_eq!(ModelType::from_str("kling_2p5_turbo_pro").unwrap(), ModelType::Kling2p5TurboPro); assert_eq!(ModelType::from_str("seedance_1p0_lite").unwrap(), ModelType::Seedance10Lite); assert_eq!(ModelType::from_str("seedance_1p0_pro").unwrap(), ModelType::Seedance10Pro); assert_eq!(ModelType::from_str("sora_2").unwrap(), ModelType::Sora2); @@ -409,7 +417,7 @@ mod tests { #[test] fn all_variants() { let mut variants = ModelType::all_variants(); - assert_eq!(variants.len(), 38); + assert_eq!(variants.len(), 39); // Image models assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell)); @@ -442,6 +450,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(ModelType::Kling16Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Kling21Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Kling21Master)); + assert_eq!(variants.pop_first(), Some(ModelType::Kling2p5TurboPro)); assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Lite)); assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Sora2)); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index ad5d429cef..5c5501eba1 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -5,4 +5,5 @@ pub mod generate_seedance_1_0_lite_image_to_video_handler; pub mod generate_seedance_1_0_pro_image_to_video_handler; pub mod generate_veo_2_image_to_video_handler; pub mod generate_veo_3_fast_image_to_video_handler; -pub mod generate_veo_3_image_to_video_handler; \ No newline at end of file +pub mod generate_veo_3_image_to_video_handler; +pub mod multi_function; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen_handler.rs new file mode 100644 index 0000000000..c677ce7795 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen_handler.rs @@ -0,0 +1,317 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::multi_function::kling_2_5_turbo_multi_function_video_gen::{Kling2p5TurboProMultiFunctionVideoGenAspectRatio, Kling2p5TurboProMultiFunctionVideoGenDuration, Kling2p5TurboProMultiFunctionVideoGenRequest, Kling2p5TurboProMultiFunctionVideoGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1p5_edit_image_webhook::{enqueue_gpt_image_1p5_image_edit_webhook, EnqueueGptImage1p5EditImageArgs, EnqueueGptImage1p5EditImageBackground, EnqueueGptImage1p5EditImageInputFidelity, EnqueueGptImage1p5EditImageNumImages, EnqueueGptImage1p5EditImageQuality, EnqueueGptImage1p5EditImageSize}; +use fal_client::requests::webhook::image::text::enqueue_gpt_image_1p5_text_to_image_webhook::{enqueue_gpt_image_1p5_text_to_image_webhook, EnqueueGptImage1p5TextToImageArgs, EnqueueGptImage1p5TextToImageBackground, EnqueueGptImage1p5TextToImageNumImages, EnqueueGptImage1p5TextToImageQuality, EnqueueGptImage1p5TextToImageSize}; +use fal_client::requests::webhook::video::image::enqueue_kling_v2p5_turbo_pro_image_to_video_webhook::{enqueue_kling_v2p5_turbo_pro_image_to_video_webhook, EnqueueKlingV2p5TurboProImageToVideoArgs, EnqueueKlingV2p5TurboProImageToVideoDurationSeconds}; +use fal_client::requests::webhook::video::text::enqueue_kling_v2p5_turbo_pro_text_to_video_webhook::{enqueue_kling_v2p5_turbo_pro_text_to_video_webhook, EnqueueKlingV2p5TurboProTextToVideoArgs, EnqueueKlingV2p5TurboProTextToVideoAspectRatio, EnqueueKlingV2p5TurboProTextToVideoDurationSeconds}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Kling 2.5 Turbo Pro Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/kling_2p5_turbo_pro", + responses( + (status = 200, description = "Success", body = Kling2p5TurboProMultiFunctionVideoGenResponse), + ), + params( + ("request" = Kling2p5TurboProMultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn kling_2p5_turbo_pro_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // NB: Masks are only relevant when this is an image editing exercise. + tokens.push(end_frame_token.to_owned()); + } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + } + } + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + if let Some(start_frame_url) = maybe_start_frame_image_url { + info!("image-to-video case"); + + let duration = match request.duration { + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Five, + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Ten, + None => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Five, + }; + + let args = EnqueueKlingV2p5TurboProImageToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").map(|s| s.to_string()), + image_url: start_frame_url, + tail_image_url: maybe_end_frame_image_url, + negative_prompt: request.negative_prompt, + duration, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_kling_v2p5_turbo_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_v2p5_turbo_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Five, + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Ten, + None => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Five, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::Square) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::Square, + Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::SixteenByNine, + Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::NineBySixteen, + None => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::Square, + }; + + let args = EnqueueKlingV2p5TurboProTextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + negative_prompt: request.negative_prompt, + duration, + aspect_ratio, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_kling_v2p5_turbo_pro_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_v2p5_turbo_pro_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Kling2p5TurboPro), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.start_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + if let Some(media_token) = &request.end_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidEndFrame, + }); + } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Kling2p5TurboProMultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/mod.rs new file mode 100644 index 0000000000..a2bfe3eba7 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/mod.rs @@ -0,0 +1 @@ +pub mod kling_2p5_turbo_pro_multi_function_video_gen_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs index 76f3f78cf0..b2d9cf458d 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs @@ -25,6 +25,7 @@ use crate::http_server::endpoints::generate::video::generate_seedance_1_0_pro_im use crate::http_server::endpoints::generate::video::generate_veo_2_image_to_video_handler::generate_veo_2_image_to_video_handler; use crate::http_server::endpoints::generate::video::generate_veo_3_fast_image_to_video_handler::generate_veo_3_fast_image_to_video_handler; use crate::http_server::endpoints::generate::video::generate_veo_3_image_to_video_handler::generate_veo_3_image_to_video_handler; +use crate::http_server::endpoints::generate::video::multi_function::kling_2p5_turbo_pro_multi_function_video_gen_handler::kling_2p5_turbo_pro_multi_function_video_gen_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; @@ -119,6 +120,12 @@ where ) ) .service(web::scope("/video") + .service(web::scope("/multi_function") + .service(web::resource("/kling_2p5_turbo_pro") + .route(web::post().to(kling_2p5_turbo_pro_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) .service(web::resource("/kling_1.6_pro_image_to_video") .route(web::post().to(generate_kling_1_6_pro_video_handler)) .route(web::head().to(|| HttpResponse::Ok()))