From e7e816b3c2fa4c22e4bde3ada46ffaac2b8e0379 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 26 Aug 2025 02:04:01 -0400 Subject: [PATCH] start adding video end frames: kling 1.6 --- ...eue_kling_16_pro_image_to_video_webhook.rs | 12 ++- .../enqueue_image_to_video_command.rs | 20 ++-- .../generic/handle_video_artcraft.rs | 1 + .../generate_kling_1_6_pro_image_to_video.rs | 6 +- .../generate_kling_1_6_pro_video_handler.rs | 91 +++++++++++-------- .../lookup/fetch_all_required_media_files.rs | 37 ++++++++ .../storyteller_web/src/util/lookup/mod.rs | 1 + .../web/storyteller_web/src/util/mod.rs | 2 + .../src/util/traits/into_media_links_trait.rs | 28 ++++++ .../storyteller_web/src/util/traits/mod.rs | 1 + 10 files changed, 150 insertions(+), 49 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/util/lookup/fetch_all_required_media_files.rs create mode 100644 crates/service/web/storyteller_web/src/util/lookup/mod.rs create mode 100644 crates/service/web/storyteller_web/src/util/traits/into_media_links_trait.rs create mode 100644 crates/service/web/storyteller_web/src/util/traits/mod.rs diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs index 9e6f9a72a3..3baaa5babd 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs @@ -5,8 +5,9 @@ use fal::endpoints::fal_ai::kling_video::v1_6::pro::image_to_video::{image_to_vi use fal::webhook::WebhookResponse; use reqwest::IntoUrl; -pub struct Kling16ProArgs<'a, U: IntoUrl, V: IntoUrl> { +pub struct Kling16ProArgs<'a, U: IntoUrl, T: IntoUrl, V: IntoUrl> { pub image_url: U, + pub end_frame_image_url: Option, pub webhook_url: V, pub prompt: &'a str, pub api_key: &'a FalApiKey, @@ -28,8 +29,8 @@ pub enum Kling16ProAspectRatio { TallNineSixteen, // 9:16 } -pub async fn enqueue_kling_16_pro_image_to_video_webhook( - args: Kling16ProArgs<'_, U, V> +pub async fn enqueue_kling_16_pro_image_to_video_webhook( + args: Kling16ProArgs<'_, U, T, V> ) -> Result { let duration = match args.duration { Kling16Duration::Default => None, @@ -45,15 +46,18 @@ pub async fn enqueue_kling_16_pro_image_to_video_webhook let image_url = args.image_url.as_str().to_string(); + let tail_image_url = args.end_frame_image_url + .map(|url| url.as_str().to_string()); + let request = ProImageToVideoRequest { image_url, + tail_image_url, prompt: args.prompt.to_string(), aspect_ratio, duration, // Maybe expose these later cfg_scale: None, negative_prompt: None, - tail_image_url: None, }; let result = image_to_video(request) diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs index 88d6edb11e..28e159b077 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs @@ -23,14 +23,22 @@ use tokens::tokens::media_files::MediaFileToken; #[derive(Deserialize)] pub struct EnqueueImageToVideoRequest { - /// Image media file; the image to remove the background from. - /// TODO: In the future we may support base64 images, URLs, or file paths here. - pub image_media_token: Option, - + /// REQUIRED. /// The model to use. pub model: Option, - - /// Optional text prompt. + + /// Currently REQUIRED. + /// Image media file; the starting frame of the video. + /// TODO: In the future we may support base64 images, URLs, or file paths here. + pub image_media_token: Option, + + /// Optional. + /// Image media file; the image to remove the background from. + /// TODO: In the future we may support base64 images, URLs, or file paths here. + pub end_frame_image_media_token: Option, + + /// Optional. + /// Text prompt used to direct the video. pub prompt: Option, } diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs index 38a91fece7..3326dc1430 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs @@ -76,6 +76,7 @@ pub async fn handle_video_artcraft( let request = GenerateKling16ProImageToVideoRequest { uuid_idempotency_token, media_file_token: request.image_media_token, + end_frame_image_media_token: request.end_frame_image_media_token, aspect_ratio: Some(GenerateKling16ProAspectRatio::WideSixteenNine), prompt: request.prompt, duration: None, diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs index 9807530180..a678c5c325 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_pro_image_to_video.rs @@ -13,7 +13,11 @@ pub struct GenerateKling16ProImageToVideoRequest { /// Source of the image file to convert to video. /// It must be an image. pub media_file_token: Option, - + + /// Optional. + /// Image media file; the image to remove the background from. + pub end_frame_image_media_token: Option, + /// Optional text prompt. pub prompt: Option, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs index ed9b11838c..8823a8fe13 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_pro_video_handler.rs @@ -5,6 +5,8 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; use crate::state::server_state::ServerState; +use crate::util::lookup::fetch_all_required_media_files::fetch_all_required_media_files; +use crate::util::traits::into_media_links_trait::IntoMediaLinks; use actix_web::web::Json; use actix_web::{web, HttpRequest}; use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProAspectRatio, GenerateKling16ProImageToVideoRequest}; @@ -24,6 +26,7 @@ use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::batch_get_media_files_by_tokens_with_connection; use mysql_queries::queries::media_files::get::get_media_file::get_media_file_with_connection; use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; use sqlx::Acquire; @@ -73,14 +76,35 @@ pub async fn generate_kling_1_6_pro_video_handler( // } //}; - let media_file_token = match &request.media_file_token { + let start_frame_media_file_token = match &request.media_file_token { Some(token) => token, None => { warn!("No media file token provided"); return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); } }; - + + let mut tokens = vec![ + start_frame_media_file_token.clone(), + ]; + + let maybe_end_frame_image_media_token = request.end_frame_image_media_token.as_ref(); + + if let Some(end_frame_token) = maybe_end_frame_image_media_token { + tokens.push(end_frame_token.clone()); + } + + let media_files = fetch_all_required_media_files( + &mut mysql_connection, + &tokens, + ).await?; + + for media_file in media_files.iter() { + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + } + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { return Err(CommonWebError::BadInputWithSimpleMessage(reason)); } @@ -91,42 +115,30 @@ pub async fn generate_kling_1_6_pro_video_handler( error!("Error inserting idempotency token: {:?}", err); CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) })?; - const IS_MOD : bool = false; - - let media_file_lookup_result = get_media_file_with_connection( - media_file_token, - IS_MOD, - &mut mysql_connection, - ).await; - let media_file = match media_file_lookup_result { - Ok(Some(media_file)) => media_file, - Ok(None) => { - warn!("MediaFile not found: {:?}", media_file_token); - return Err(CommonWebError::NotFound); - }, - Err(err) => { - warn!("Error looking up media_file: {:?}", err); - return Err(CommonWebError::ServerError); - } + let media_domain = get_media_domain(&http_request); + + let start_frame_url = media_files.iter() + .find(|file| &file.token == start_frame_media_file_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("Start frame media file not found after fetch"); + CommonWebError::NotFound + })?; + + let maybe_end_frame_url = match maybe_end_frame_image_media_token { + None => None, + Some(end_frame_token) => Some(media_files.iter() + .find(|file| &file.token == end_frame_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("End frame media file not found after fetch"); + CommonWebError::NotFound + })?), }; - if !media_file.media_type.is_jpg_or_png_or_legacy_image() { - return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); - } - - let media_domain = get_media_domain(&http_request); - - let bucket_path = MediaFileBucketPath::from_object_hash( - &media_file.public_bucket_directory_hash, - media_file.maybe_public_bucket_prefix.as_deref(), - media_file.maybe_public_bucket_extension.as_deref()); - - let media_links = MediaLinksBuilder::from_media_path_and_env( - media_domain, - server_state.server_environment, - &bucket_path); - info!("Fal webhook URL: {}", server_state.fal.webhook_url); let prompt = request.prompt @@ -146,9 +158,10 @@ pub async fn generate_kling_1_6_pro_video_handler( Some(GenerateKling16ProDuration::TenSeconds) => Kling16Duration::TenSeconds, None => Kling16Duration::FiveSeconds, }; - + let args = Kling16ProArgs { - image_url: media_links.cdn_url, + image_url: start_frame_url, + end_frame_image_url: maybe_end_frame_url, webhook_url: &server_state.fal.webhook_url, duration, prompt, @@ -159,7 +172,7 @@ pub async fn generate_kling_1_6_pro_video_handler( let fal_result = enqueue_kling_16_pro_image_to_video_webhook(args) .await .map_err(|err| { - warn!("Error calling remove_background_rembg_webhook: {:?}", err); + warn!("Error calling enqueue_kling_16_pro_image_to_video_webhook: {:?}", err); CommonWebError::ServerError })?; @@ -206,6 +219,8 @@ pub async fn generate_kling_1_6_pro_video_handler( } }; + // TODO(bt,2025-08-26): Save reference images. + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { uuid_idempotency_token: &request.uuid_idempotency_token, maybe_external_third_party_id: &external_job_id, diff --git a/crates/service/web/storyteller_web/src/util/lookup/fetch_all_required_media_files.rs b/crates/service/web/storyteller_web/src/util/lookup/fetch_all_required_media_files.rs new file mode 100644 index 0000000000..8808f6009d --- /dev/null +++ b/crates/service/web/storyteller_web/src/util/lookup/fetch_all_required_media_files.rs @@ -0,0 +1,37 @@ +use crate::http_server::common_responses::common_web_error::CommonWebError; +use log::{error, warn}; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens_with_connection, MediaFilesByTokensRecord}; +use sqlx::pool::PoolConnection; +use sqlx::MySql; +use tokens::tokens::media_files::MediaFileToken; + +const CAN_SEE_DELETED: bool = false; + +/// Look up all the media tokens. If any are missing, return an error. +pub async fn fetch_all_required_media_files( + mysql_connection: &mut PoolConnection, + media_file_tokens: &[MediaFileToken], +) -> Result, CommonWebError> { + let result = batch_get_media_files_by_tokens_with_connection( + mysql_connection, + &media_file_tokens, + CAN_SEE_DELETED, + ).await; + + let media_files = match result { + Ok(files) => files, + Err(err) => { + error!("Error getting media files by tokens: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if media_files.len() != media_file_tokens.len() { + warn!("Wrong number of media files returned for tokens: {} found for {} tokens", media_files.len(), media_file_tokens.len()); + return Err(CommonWebError::BadInputWithSimpleMessage( + format!("Not all media files could be found. Media files found: {}, tokens provided: {}", + media_files.len(), media_file_tokens.len()))); + } + + Ok(media_files) +} diff --git a/crates/service/web/storyteller_web/src/util/lookup/mod.rs b/crates/service/web/storyteller_web/src/util/lookup/mod.rs new file mode 100644 index 0000000000..e31f48b51a --- /dev/null +++ b/crates/service/web/storyteller_web/src/util/lookup/mod.rs @@ -0,0 +1 @@ +pub mod fetch_all_required_media_files; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/util/mod.rs b/crates/service/web/storyteller_web/src/util/mod.rs index ad33881fae..6ca3449ae4 100644 --- a/crates/service/web/storyteller_web/src/util/mod.rs +++ b/crates/service/web/storyteller_web/src/util/mod.rs @@ -11,7 +11,9 @@ pub mod enroll_in_studio; pub mod generate_random_username; pub mod http_download_url_to_bytes; pub mod http_download_url_to_tempfile; +pub mod lookup; pub mod placeholder_images; pub mod read_toml_file_to_struct; pub mod title_to_url_slug; +pub mod traits; pub mod troll_user_bans; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/util/traits/into_media_links_trait.rs b/crates/service/web/storyteller_web/src/util/traits/into_media_links_trait.rs new file mode 100644 index 0000000000..054ada52c8 --- /dev/null +++ b/crates/service/web/storyteller_web/src/util/traits/into_media_links_trait.rs @@ -0,0 +1,28 @@ +use crate::http_server::common_responses::media::media_domain::MediaDomain; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use artcraft_api_defs::common::responses::media_links::MediaLinks; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::MediaFilesByTokensRecord; +use server_environment::ServerEnvironment; + +pub trait IntoMediaLinks { + fn to_media_links( + &self, + media_domain: MediaDomain, + server_environment: ServerEnvironment, + ) -> MediaLinks; +} + +impl IntoMediaLinks for MediaFilesByTokensRecord { + fn to_media_links(&self, media_domain: MediaDomain, server_environment: ServerEnvironment) -> MediaLinks { + let bucket_path = MediaFileBucketPath::from_object_hash( + &self.public_bucket_directory_hash, + self.maybe_public_bucket_prefix.as_deref(), + self.maybe_public_bucket_extension.as_deref()); + MediaLinksBuilder::from_media_path_and_env( + media_domain, + server_environment, + &bucket_path + ) + } +} diff --git a/crates/service/web/storyteller_web/src/util/traits/mod.rs b/crates/service/web/storyteller_web/src/util/traits/mod.rs new file mode 100644 index 0000000000..1edbed77c5 --- /dev/null +++ b/crates/service/web/storyteller_web/src/util/traits/mod.rs @@ -0,0 +1 @@ +pub mod into_media_links_trait; \ No newline at end of file