kling 2.1 pro

crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_2_1_pro_image_to_video.rs

@@ -0,0 +1,49 @@
+use serde::{Deserialize, Serialize};
+use tokens::tokens::generic_inference_jobs::InferenceJobToken;
+use tokens::tokens::media_files::MediaFileToken;
+use utoipa::ToSchema;
+
+pub const GENERATE_KLING_2_1_PRO_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/kling_2.1_pro_image_to_video";
+
+#[derive(Serialize, Deserialize, ToSchema)]
+pub struct GenerateKling21ProImageToVideoRequest {
+  /// Idempotency token to prevent duplicate requests.
+  pub uuid_idempotency_token: String,
+  
+  /// Source of the image file to convert to video.
+  /// It must be an image.
+  pub media_file_token: Option<MediaFileToken>,
+  
+  /// Optional text prompt.
+  pub prompt: Option<String>,
+  
+  /// Optional: aspect ratio of the generated video.
+  pub aspect_ratio: Option<GenerateKling21ProAspectRatio>,
+
+  /// Optional: aspect ratio of the generated video.
+  pub duration: Option<GenerateKling21ProDuration>,
+}
+
+#[derive(Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum GenerateKling21ProAspectRatio {
+  /// 16:9 aspect ratio
+  WideSixteenNine,
+  /// 9:16 aspect ratio
+  TallNineSixteen,
+  /// 1:1 aspect ratio
+  Square,
+}
+
+#[derive(Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum GenerateKling21ProDuration {
+  FiveSeconds,
+  TenSeconds,
+}
+
+#[derive(Serialize, Deserialize, ToSchema)]
+pub struct GenerateKling21ProImageToVideoResponse {
+  pub success: bool,
+  pub inference_job_token: InferenceJobToken,
+}

crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs

@@ -1 +1,2 @@
-pub mod generate_kling_1_6_pro_image_to_video;
+pub mod generate_kling_1_6_pro_image_to_video;
+pub mod generate_kling_2_1_pro_image_to_video;

crates/service/web/storyteller_web/src/docs/api_doc.rs

@@ -26,6 +26,10 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::G
 use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProDuration;
 use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProImageToVideoRequest;
 use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::GenerateKling16ProImageToVideoResponse;
+use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProAspectRatio;
+use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProDuration;
+use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProImageToVideoRequest;
+use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::GenerateKling21ProImageToVideoResponse;
 use billing_component::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_error::CreateCheckoutSessionError;
 use billing_component::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_json_handler::*;
 use billing_component::users::http_endpoints::list_active_user_subscriptions_handler::*;
@@ -69,7 +73,6 @@ use crate::http_server::common_requests::media_file_token_path_info::MediaFileTo
 use crate::http_server::common_responses::media::cover_image_links::CoverImageLinks;
 use crate::http_server::common_responses::media::media_file_cover_image_details::MediaFileCoverImageDetails;
 use crate::http_server::common_responses::media::media_file_cover_image_details::MediaFileDefaultCover;
-use crate::http_server::endpoints::generate::video::generate_kling_1_6_pro_video_handler::generate_kling_1_6_pro_video_handler;
 use crate::http_server::common_responses::media::media_links::*;
 use crate::http_server::common_responses::media::weights_cover_image_details::*;
 use crate::http_server::common_responses::media_file_origin_details::*;
@@ -121,6 +124,7 @@ use crate::http_server::endpoints::inference_job::list::list_session_jobs_handle
 use crate::http_server::endpoints::media_files::common_responses::live_portrait::MediaFileLivePortraitDetails;
 use crate::http_server::endpoints::media_files::delete::delete_media_file_handler::*;
 use crate::http_server::endpoints::media_files::edit::change_media_file_animation_type_handler::*;
+use crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler;
 use crate::http_server::endpoints::media_files::edit::change_media_file_engine_category_handler::*;
 use crate::http_server::endpoints::media_files::edit::change_media_file_visibility_handler::*;
 use crate::http_server::endpoints::media_files::edit::rename_media_file_handler::*;
@@ -228,6 +232,7 @@ use crate::http_server::web_utils::response_success_helpers::*;
     crate::http_server::endpoints::inference_job::get::batch_get_inference_job_status_handler::batch_get_inference_job_status_handler,
     crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::get_inference_job_status_handler,
     crate::http_server::endpoints::inference_job::list::list_session_jobs_handler::list_session_jobs_handler,
+    crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler,
     crate::http_server::endpoints::media_files::delete::delete_media_file_handler::delete_media_file_handler,
     crate::http_server::endpoints::media_files::edit::change_media_file_animation_type_handler::change_media_file_animation_type_handler,
     crate::http_server::endpoints::media_files::edit::change_media_file_engine_category_handler::change_media_file_engine_category_handler,
@@ -411,6 +416,10 @@ use crate::http_server::web_utils::response_success_helpers::*;
     CreateAccountSuccessResponse,
     CreateBetaKeysError,
     CreateBetaKeysRequest,
+    GenerateKling21ProAspectRatio,
+    GenerateKling21ProDuration,
+    GenerateKling21ProImageToVideoRequest,
+    GenerateKling21ProImageToVideoResponse,
     CreateBetaKeysSuccessResponse,
     CreateCheckoutSessionError,
     CreateCheckoutSessionRequest,

crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_2_1_pro_video_handler.rs

@@ -0,0 +1,226 @@
+use std::fmt;
+use std::sync::Arc;
+
+use crate::http_server::common_requests::media_file_token_path_info::MediaFileTokenPathInfo;
+use crate::http_server::common_responses::common_web_error::CommonWebError;
+use crate::http_server::common_responses::media::media_links::MediaLinks;
+use crate::http_server::deprecated_endpoints::engine::create_scene_handler::CreateSceneError;
+use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
+use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
+use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
+use crate::http_server::web_utils::response_success_helpers::simple_json_success;
+use crate::state::server_state::ServerState;
+use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole};
+use actix_web::error::ResponseError;
+use actix_web::http::StatusCode;
+use actix_web::web::Json;
+use actix_web::web::Path;
+use actix_web::{web, HttpRequest, HttpResponse};
+use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProAspectRatio, GenerateKling21ProImageToVideoRequest};
+use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProDuration, GenerateKling21ProImageToVideoResponse};
+use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
+use enums::common::visibility::Visibility;
+use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::enqueue_kling_21_pro_image_to_video_webhook;
+use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::Kling21ProArgs;
+use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::Kling21ProAspectRatio;
+use fal_client::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::Kling21ProDuration;
+use http_server_common::request::get_request_ip::get_request_ip;
+use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
+use idempotency::uuid::generate_random_uuid;
+use log::{error, info, warn};
+use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
+use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
+use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
+use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
+use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile};
+use tokens::tokens::media_files::MediaFileToken;
+use utoipa::ToSchema;
+
+/*
+ TODO: Either on the Tauri client or as an API, make "generic" image / video / object endpoints. 
+  Take a reference image payload: 
+  
+    ReferenceImage {
+      media_token: MediaFileToken,
+      reference_type: ReferenceType
+    } 
+    
+    enum ReferenceType {
+      PrimaryReference,
+      CharacterReference,
+      LocationReference,
+      StyleReference,
+      ...
+      GenericReference,
+    }
+    
+    // **ORDER MATTERS**
+    reference_images: Vec<ReferenceImage>,
+
+*/
+
+/// Kling 2.1 Pro Image to Video
+#[utoipa::path(
+  post,
+  tag = "Generate Videos",
+  path = "/v1/generate/video/kling_2.1_pro_image_to_video",
+  responses(
+    (status = 200, description = "Success", body = GenerateKling21ProImageToVideoResponse),
+  ),
+  params(
+    ("request" = GenerateKling21ProImageToVideoRequest, description = "Payload for Request"),
+  )
+)]
+pub async fn generate_kling_2_1_pro_video_handler(
+  http_request: HttpRequest,
+  request: Json<GenerateKling21ProImageToVideoRequest>,
+  server_state: web::Data<Arc<ServerState>>
+) -> Result<Json<GenerateKling21ProImageToVideoResponse>, CommonWebError> {
+  let maybe_user_session = server_state
+      .session_checker
+      .maybe_get_user_session(&http_request, &server_state.mysql_pool)
+      .await
+      .map_err(|e| {
+        warn!("Session checker error: {:?}", e);
+        CommonWebError::ServerError
+      })?;
+
+  let maybe_avt_token = server_state
+      .avt_cookie_manager
+      .get_avt_token_from_request(&http_request);
+
+  // TODO: Limit usage for new accounts. Billing, free credits metering, etc.
+
+  //let user_session = match maybe_user_session {
+  //  Some(session) => session,
+  //  None => {
+  //    warn!("not logged in");
+  //    return Err(RemoveImageBackgroundError::NotAuthorized);
+  //  }
+  //};
+
+  let media_file_token = match &request.media_file_token {
+    Some(token) => token,
+    None => {
+      warn!("No media file token provided");
+      return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string()));
+    }
+  };
+  
+  if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
+    return Err(CommonWebError::BadInputWithSimpleMessage(reason));
+  }
+
+  insert_idempotency_token(&request.uuid_idempotency_token, &server_state.mysql_pool)
+      .await
+      .map_err(|err| {
+        error!("Error inserting idempotency token: {:?}", err);
+        CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string())
+      })?;
+  const IS_MOD : bool = false;
+  
+  let media_file_lookup_result = get_media_file(
+    media_file_token,
+    IS_MOD,
+    &server_state.mysql_pool,
+  ).await;
+
+  let media_file = match media_file_lookup_result {
+    Ok(Some(media_file)) => media_file,
+    Ok(None) => {
+      warn!("MediaFile not found: {:?}", media_file_token);
+      return Err(CommonWebError::NotFound);
+    },
+    Err(err) => {
+      warn!("Error looking up media_file: {:?}", err);
+      return Err(CommonWebError::ServerError);
+    }
+  };
+
+  if !media_file.media_type.is_jpg_or_png_or_legacy_image() {
+    return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string()));
+  }
+  
+  let media_domain = get_media_domain(&http_request);
+  
+  let bucket_path = MediaFileBucketPath::from_object_hash(
+    &media_file.public_bucket_directory_hash,
+    media_file.maybe_public_bucket_prefix.as_deref(),
+    media_file.maybe_public_bucket_extension.as_deref());
+  
+  let media_links = MediaLinks::from_media_path_and_env(
+    media_domain, 
+    server_state.server_environment, 
+    &bucket_path);
+  
+  info!("Fal webhook URL: {}", server_state.fal.webhook_url);
+  
+  let prompt = request.prompt
+      .as_deref()
+      .map(|prompt| prompt.trim())
+      .unwrap_or_else(|| "");
+  
+  let aspect_ratio = match &request.aspect_ratio {
+    Some(GenerateKling21ProAspectRatio::Square) => Kling21ProAspectRatio::Square,
+    Some(GenerateKling21ProAspectRatio::WideSixteenNine) => Kling21ProAspectRatio::WideSixteenNine,
+    Some(GenerateKling21ProAspectRatio::TallNineSixteen) => Kling21ProAspectRatio::TallNineSixteen,
+    None => Kling21ProAspectRatio::WideSixteenNine, // Default to 16:9
+  };
+  
+  let duration = match &request.duration {
+    Some(GenerateKling21ProDuration::FiveSeconds) => Kling21ProDuration::FiveSeconds,
+    Some(GenerateKling21ProDuration::TenSeconds) => Kling21ProDuration::TenSeconds,
+    None => Kling21ProDuration::FiveSeconds, 
+  };
+  
+  let args = Kling21ProArgs {
+    image_url: media_links.cdn_url,
+    webhook_url: &server_state.fal.webhook_url,
+    duration,
+    prompt,
+    aspect_ratio,
+    api_key: &server_state.fal.api_key,
+  };
+
+  let fal_result = enqueue_kling_21_pro_image_to_video_webhook(args)
+      .await
+      .map_err(|err| {
+        warn!("Error calling enqueue_kling_21_pro_image_to_video_webhook: {:?}", err);
+        CommonWebError::ServerError
+      })?;
+
+  let external_job_id = fal_result.request_id
+      .ok_or_else(|| {
+        warn!("Fal request_id is None");
+        CommonWebError::ServerError
+      })?;
+  
+  info!("Fal request_id: {}", external_job_id);
+  
+  let ip_address = get_request_ip(&http_request);
+
+  let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
+    uuid_idempotency_token: &request.uuid_idempotency_token,
+    maybe_external_third_party_id: &external_job_id,
+    fal_category: FalCategory::VideoGeneration,
+    maybe_inference_args: None,
+    maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
+    maybe_avt_token: maybe_avt_token.as_ref(),
+    creator_ip_address: &ip_address,
+    creator_set_visibility: Visibility::Public,
+    mysql_pool: &server_state.mysql_pool,
+  }).await;
+
+  let job_token = match db_result {
+    Ok(token) => token,
+    Err(err) => {
+      warn!("Error inserting generic inference job for FAL queue: {:?}", err);
+      return Err(CommonWebError::ServerError);
+    }
+  };
+
+  Ok(Json(GenerateKling21ProImageToVideoResponse {
+    success: true,
+    inference_job_token: job_token,
+  }))
+}

crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs

@@ -1 +1,2 @@
-pub mod generate_kling_1_6_pro_video_handler;
+pub mod generate_kling_1_6_pro_video_handler;
+pub mod generate_kling_2_1_pro_video_handler;

crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs

@@ -6,6 +6,7 @@ use crate::http_server::endpoints::generate::image::remove_image_background_hand
 use crate::http_server::endpoints::generate::object::generate_hunyuan_21_image_to_3d_handler::generate_hunyuan_21_image_to_3d_handler;
 use crate::http_server::endpoints::generate::object::generate_hunyuan_2_image_to_3d_handler::generate_hunyuan_2_image_to_3d_handler;
 use crate::http_server::endpoints::generate::video::generate_kling_1_6_pro_video_handler::generate_kling_1_6_pro_video_handler;
+use crate::http_server::endpoints::generate::video::generate_kling_2_1_pro_video_handler::generate_kling_2_1_pro_video_handler;
 use actix_http::body::MessageBody;
 use actix_service::ServiceFactory;
 use actix_web::dev::{ServiceRequest, ServiceResponse};
@@ -50,6 +51,10 @@ where
               .route(web::post().to(generate_kling_1_6_pro_video_handler))
               .route(web::head().to(|| HttpResponse::Ok()))
           )
+          .service(web::resource("/kling_2.1_pro_image_to_video")
+              .route(web::post().to(generate_kling_2_1_pro_video_handler))
+              .route(web::head().to(|| HttpResponse::Ok()))
+          )
       )
       .service(web::scope("/object")
           .service(web::resource("/hunyuan_2_image_to_3d")