diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs index 6fc471584b..f52a882212 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs @@ -5,8 +5,9 @@ use fal::endpoints::fal_ai::kling_video::v2_1::pro::image_to_video::{image_to_vi use fal::webhook::WebhookResponse; use reqwest::IntoUrl; -pub struct Kling21ProArgs<'a, U: IntoUrl, V: IntoUrl> { +pub struct Kling21ProArgs<'a, U: IntoUrl, T: IntoUrl, V: IntoUrl> { pub image_url: U, + pub end_frame_image_url: Option, pub webhook_url: V, pub prompt: &'a str, pub api_key: &'a FalApiKey, @@ -28,8 +29,8 @@ pub enum Kling21ProAspectRatio { TallNineSixteen, // 9:16 } -pub async fn enqueue_kling_21_pro_image_to_video_webhook( - args: Kling21ProArgs<'_, U, V> +pub async fn enqueue_kling_21_pro_image_to_video_webhook( + args: Kling21ProArgs<'_, U, T, V> ) -> Result { let duration = match args.duration { Kling21ProDuration::Default => None, @@ -45,15 +46,18 @@ pub async fn enqueue_kling_21_pro_image_to_video_webhook let image_url = args.image_url.as_str().to_string(); + let tail_image_url = args.end_frame_image_url + .map(|url| url.as_str().to_string()); + let request = ProImageToVideoRequest { image_url, prompt: args.prompt.to_string(), + tail_image_url, aspect_ratio, duration, // Maybe expose these later cfg_scale: None, negative_prompt: None, - tail_image_url: None, }; let result = image_to_video(request) @@ -64,13 +68,13 @@ pub async fn enqueue_kling_21_pro_image_to_video_webhook result.map_err(|err| classify_fal_error(err)) } - #[cfg(test)] mod tests { use crate::creds::fal_api_key::FalApiKey; use crate::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::{enqueue_kling_21_pro_image_to_video_webhook, Kling21ProArgs, Kling21ProAspectRatio, Kling21ProDuration}; use errors::AnyhowResult; use std::fs::read_to_string; + use test_data::web::image_urls::{JUNO_AT_LAKE_IMAGE_URL, WIDE_FALL_MOUNTAINS_IMAGE_URL}; #[tokio::test] #[ignore] @@ -83,8 +87,9 @@ mod tests { let api_key = FalApiKey::from_str(&secret); let args = Kling21ProArgs { - image_url: image_url, - prompt: "a shot of the mountains, the camera rotates around to show the mountain range", + image_url: WIDE_FALL_MOUNTAINS_IMAGE_URL, + end_frame_image_url: Some(JUNO_AT_LAKE_IMAGE_URL.to_string()), + prompt: "a shot of the mountains, the camera pulls back to show a corgi waiting to jump into the lake", api_key: &api_key, duration: Kling21ProDuration::Default, aspect_ratio: Kling21ProAspectRatio::WideSixteenNine, diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs index 8faf3850da..677bc813cc 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs @@ -13,6 +13,10 @@ pub struct GenerateKling21ProImageToVideoRequest { /// Source of the image file to convert to video. /// It must be an image. pub media_file_token: Option, + + /// Optional. + /// Image media file; the image to remove the background from. + pub end_frame_image_media_token: Option, /// Optional text prompt. pub prompt: Option, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs index cb87bb8b55..84d1c5c670 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs @@ -28,6 +28,8 @@ use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, g use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; use sqlx::Acquire; use utoipa::ToSchema; +use crate::util::lookup::fetch_all_required_media_files::fetch_all_required_media_files; +use crate::util::traits::into_media_links_trait::IntoMediaLinks; /// Kling 2.1 Pro Image to Video #[utoipa::path( @@ -73,14 +75,35 @@ pub async fn generate_kling_2_1_pro_video_handler( // } //}; - let media_file_token = match &request.media_file_token { + let start_frame_media_file_token = match &request.media_file_token { Some(token) => token, None => { warn!("No media file token provided"); return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); } }; - + + let mut tokens = vec![ + start_frame_media_file_token.clone(), + ]; + + let maybe_end_frame_image_media_token = request.end_frame_image_media_token.as_ref(); + + if let Some(end_frame_token) = maybe_end_frame_image_media_token { + tokens.push(end_frame_token.clone()); + } + + let media_files = fetch_all_required_media_files( + &mut mysql_connection, + &tokens, + ).await?; + + for media_file in media_files.iter() { + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + } + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { return Err(CommonWebError::BadInputWithSimpleMessage(reason)); } @@ -91,42 +114,30 @@ pub async fn generate_kling_2_1_pro_video_handler( error!("Error inserting idempotency token: {:?}", err); CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) })?; - const IS_MOD : bool = false; - - let media_file_lookup_result = get_media_file_with_connection( - media_file_token, - IS_MOD, - &mut mysql_connection, - ).await; - let media_file = match media_file_lookup_result { - Ok(Some(media_file)) => media_file, - Ok(None) => { - warn!("MediaFile not found: {:?}", media_file_token); - return Err(CommonWebError::NotFound); - }, - Err(err) => { - warn!("Error looking up media_file: {:?}", err); - return Err(CommonWebError::ServerError); - } + let media_domain = get_media_domain(&http_request); + + let start_frame_url = media_files.iter() + .find(|file| &file.token == start_frame_media_file_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("Start frame media file not found after fetch"); + CommonWebError::NotFound + })?; + + let maybe_end_frame_url = match maybe_end_frame_image_media_token { + None => None, + Some(end_frame_token) => Some(media_files.iter() + .find(|file| &file.token == end_frame_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("End frame media file not found after fetch"); + CommonWebError::NotFound + })?), }; - if !media_file.media_type.is_jpg_or_png_or_legacy_image() { - return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); - } - - let media_domain = get_media_domain(&http_request); - - let bucket_path = MediaFileBucketPath::from_object_hash( - &media_file.public_bucket_directory_hash, - media_file.maybe_public_bucket_prefix.as_deref(), - media_file.maybe_public_bucket_extension.as_deref()); - - let media_links = MediaLinksBuilder::from_media_path_and_env( - media_domain, - server_state.server_environment, - &bucket_path); - info!("Fal webhook URL: {}", server_state.fal.webhook_url); let prompt = request.prompt @@ -148,7 +159,8 @@ pub async fn generate_kling_2_1_pro_video_handler( }; let args = Kling21ProArgs { - image_url: media_links.cdn_url, + image_url: start_frame_url, + end_frame_image_url: maybe_end_frame_url, webhook_url: &server_state.fal.webhook_url, duration, prompt,