diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs new file mode 100644 index 0000000000..8faf3850da --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs @@ -0,0 +1,49 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GENERATE_KLING_2_1_PRO_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/kling_2.1_pro_image_to_video"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateKling21ProImageToVideoRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Source of the image file to convert to video. + /// It must be an image. + pub media_file_token: Option, + + /// Optional text prompt. + pub prompt: Option, + + /// Optional: aspect ratio of the generated video. + pub aspect_ratio: Option, + + /// Optional: aspect ratio of the generated video. + pub duration: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateKling21ProAspectRatio { + /// 16:9 aspect ratio + WideSixteenNine, + /// 9:16 aspect ratio + TallNineSixteen, + /// 1:1 aspect ratio + Square, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateKling21ProDuration { + FiveSeconds, + TenSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateKling21ProImageToVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs index 5a1956a8f5..eaf6285789 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs @@ -1 +1,2 @@ -pub mod generate_kling_1_6_pro_image_to_video; \ No newline at end of file +pub mod generate_kling_1_6_pro_image_to_video; +pub mod generate_kling_2_1_pro_image_to_video; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index ece164e702..29a5025b0e 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -26,6 +26,10 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::G use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProDuration; use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProImageToVideoRequest; use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProImageToVideoResponse; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProAspectRatio; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProDuration; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProImageToVideoRequest; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProImageToVideoResponse; use billing_component::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError; use billing_component::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_json_handler::*; use billing_component::users::http_endpoints::list_active_user_subscriptions_handler::*; @@ -69,7 +73,6 @@ use crate::http_server::common_requests::media_file_token_path_info::MediaFileTo use crate::http_server::common_responses::media::cover_image_links::CoverImageLinks; use crate::http_server::common_responses::media::media_file_cover_image_details::MediaFileCoverImageDetails; use crate::http_server::common_responses::media::media_file_cover_image_details::MediaFileDefaultCover; -use crate::http_server::endpoints::generate::video::generate_kling_1_6_pro_video_handler::generate_kling_1_6_pro_video_handler; use crate::http_server::common_responses::media::media_links::*; use crate::http_server::common_responses::media::weights_cover_image_details::*; use crate::http_server::common_responses::media_file_origin_details::*; @@ -121,6 +124,7 @@ use crate::http_server::endpoints::inference_job::list::list_session_jobs_handle use crate::http_server::endpoints::media_files::common_responses::live_portrait::MediaFileLivePortraitDetails; use crate::http_server::endpoints::media_files::delete::delete_media_file_handler::*; use crate::http_server::endpoints::media_files::edit::change_media_file_animation_type_handler::*; +use crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler; use crate::http_server::endpoints::media_files::edit::change_media_file_engine_category_handler::*; use crate::http_server::endpoints::media_files::edit::change_media_file_visibility_handler::*; use crate::http_server::endpoints::media_files::edit::rename_media_file_handler::*; @@ -228,6 +232,7 @@ use crate::http_server::web_utils::response_success_helpers::*; crate::http_server::endpoints::inference_job::get::batch_get_inference_job_status_handler::batch_get_inference_job_status_handler, crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::get_inference_job_status_handler, crate::http_server::endpoints::inference_job::list::list_session_jobs_handler::list_session_jobs_handler, + crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler, crate::http_server::endpoints::media_files::delete::delete_media_file_handler::delete_media_file_handler, crate::http_server::endpoints::media_files::edit::change_media_file_animation_type_handler::change_media_file_animation_type_handler, crate::http_server::endpoints::media_files::edit::change_media_file_engine_category_handler::change_media_file_engine_category_handler, @@ -411,6 +416,10 @@ use crate::http_server::web_utils::response_success_helpers::*; CreateAccountSuccessResponse, CreateBetaKeysError, CreateBetaKeysRequest, + GenerateKling21ProAspectRatio, + GenerateKling21ProDuration, + GenerateKling21ProImageToVideoRequest, + GenerateKling21ProImageToVideoResponse, CreateBetaKeysSuccessResponse, CreateCheckoutSessionError, CreateCheckoutSessionRequest, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs new file mode 100644 index 0000000000..7870b5ba05 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs @@ -0,0 +1,226 @@ +use std::fmt; +use std::sync::Arc; + +use crate::http_server::common_requests::media_file_token_path_info::MediaFileTokenPathInfo; +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links::MediaLinks; +use crate::http_server::deprecated_endpoints::engine::create_scene_handler::CreateSceneError; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::state::server_state::ServerState; +use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Json; +use actix_web::web::Path; +use actix_web::{web, HttpRequest, HttpResponse}; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProAspectRatio, GenerateKling21ProImageToVideoRequest}; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProDuration, GenerateKling21ProImageToVideoResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::enqueue_kling_21_pro_image_to_video_webhook; +use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::Kling21ProArgs; +use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::Kling21ProAspectRatio; +use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::Kling21ProDuration; +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use idempotency::uuid::generate_random_uuid; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/* + TODO: Either on the Tauri client or as an API, make "generic" image / video / object endpoints. + Take a reference image payload: + + ReferenceImage { + media_token: MediaFileToken, + reference_type: ReferenceType + } + + enum ReferenceType { + PrimaryReference, + CharacterReference, + LocationReference, + StyleReference, + ... + GenericReference, + } + + // **ORDER MATTERS** + reference_images: Vec, + +*/ + +/// Kling 2.1 Pro Image to Video +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/kling_2.1_pro_image_to_video", + responses( + (status = 200, description = "Success", body = GenerateKling21ProImageToVideoResponse), + ), + params( + ("request" = GenerateKling21ProImageToVideoRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_kling_2_1_pro_video_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(RemoveImageBackgroundError::NotAuthorized); + // } + //}; + + let media_file_token = match &request.media_file_token { + Some(token) => token, + None => { + warn!("No media file token provided"); + return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); + } + }; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &server_state.mysql_pool) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) + })?; + const IS_MOD : bool = false; + + let media_file_lookup_result = get_media_file( + media_file_token, + IS_MOD, + &server_state.mysql_pool, + ).await; + + let media_file = match media_file_lookup_result { + Ok(Some(media_file)) => media_file, + Ok(None) => { + warn!("MediaFile not found: {:?}", media_file_token); + return Err(CommonWebError::NotFound); + }, + Err(err) => { + warn!("Error looking up media_file: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + + let media_domain = get_media_domain(&http_request); + + let bucket_path = MediaFileBucketPath::from_object_hash( + &media_file.public_bucket_directory_hash, + media_file.maybe_public_bucket_prefix.as_deref(), + media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinks::from_media_path_and_env( + media_domain, + server_state.server_environment, + &bucket_path); + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let prompt = request.prompt + .as_deref() + .map(|prompt| prompt.trim()) + .unwrap_or_else(|| ""); + + let aspect_ratio = match &request.aspect_ratio { + Some(GenerateKling21ProAspectRatio::Square) => Kling21ProAspectRatio::Square, + Some(GenerateKling21ProAspectRatio::WideSixteenNine) => Kling21ProAspectRatio::WideSixteenNine, + Some(GenerateKling21ProAspectRatio::TallNineSixteen) => Kling21ProAspectRatio::TallNineSixteen, + None => Kling21ProAspectRatio::WideSixteenNine, // Default to 16:9 + }; + + let duration = match &request.duration { + Some(GenerateKling21ProDuration::FiveSeconds) => Kling21ProDuration::FiveSeconds, + Some(GenerateKling21ProDuration::TenSeconds) => Kling21ProDuration::TenSeconds, + None => Kling21ProDuration::FiveSeconds, + }; + + let args = Kling21ProArgs { + image_url: media_links.cdn_url, + webhook_url: &server_state.fal.webhook_url, + duration, + prompt, + aspect_ratio, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_kling_21_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_21_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + Ok(Json(GenerateKling21ProImageToVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index d7a0e16804..2dd59909cb 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -1 +1,2 @@ -pub mod generate_kling_1_6_pro_video_handler; \ No newline at end of file +pub mod generate_kling_1_6_pro_video_handler; +pub mod generate_kling_2_1_pro_video_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index cb06a16df1..48a4e5f5a5 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -6,6 +6,7 @@ use crate::http_server::endpoints::generate::image::remove_image_background_hand use crate::http_server::endpoints::generate::object::generate_hunyuan_21_image_to_3d_handler::generate_hunyuan_21_image_to_3d_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_image_to_3d_handler::generate_hunyuan_2_image_to_3d_handler; use crate::http_server::endpoints::generate::video::generate_kling_1_6_pro_video_handler::generate_kling_1_6_pro_video_handler; +use crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; @@ -50,6 +51,10 @@ where .route(web::post().to(generate_kling_1_6_pro_video_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/kling_2.1_pro_image_to_video") + .route(web::post().to(generate_kling_2_1_pro_video_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) .service(web::scope("/object") .service(web::resource("/hunyuan_2_image_to_3d")