diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/mod.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/mod.rs index b1b6b4f003..82a9d3c9e0 100644 --- a/crates/lib/api_clients/fal_client/src/requests/webhook/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/mod.rs @@ -1 +1,2 @@ -pub mod image; \ No newline at end of file +pub mod image; +pub mod video; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_image_to_video_webhook.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_image_to_video_webhook.rs new file mode 100644 index 0000000000..345c12c5da --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_image_to_video_webhook.rs @@ -0,0 +1,83 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::endpoints::fal_ai::kling_video::v1_6::pro::image_to_video::{image_to_video, ProImageToVideoRequest}; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Kling16Args<'a, U: IntoUrl, V: IntoUrl> { + pub image_url: U, + pub webhook_url: V, + pub prompt: &'a str, + pub api_key: &'a FalApiKey, + pub duration: Kling16Duration, +} + +#[derive(Copy, Clone, Debug)] +pub enum Kling16Duration { + Default, + FiveSeconds, + TenSeconds, +} + +pub async fn enqueue_kling_16_image_to_video_webhook( + args: Kling16Args<'_, U, V> +) -> Result { + let duration = match args.duration { + Kling16Duration::Default => None, + Kling16Duration::FiveSeconds => Some("5".to_string()), // Gross... + Kling16Duration::TenSeconds => Some("10".to_string()), + }; + + let image_url = args.image_url.as_str().to_string(); + + let request = ProImageToVideoRequest { + image_url, + prompt: args.prompt.to_string(), + aspect_ratio: None, + cfg_scale: None, + duration, + negative_prompt: None, + tail_image_url: None, + }; + + let result = image_to_video(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +/* +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::{enqueue_kling_16_image_to_video_webhook, Kling16Args, Kling16Duration}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; + + #[tokio::test] + #[ignore] + async fn test_kling16_video() -> AnyhowResult<()> { + let image = test_file_path("test_data/image/juno.jpg")?; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Kling16Args { + image_path: image, + prompt: "a corgi looks out over the water", + api_key: &api_key, + duration: Kling16Duration::Default, + }; + + let result = enqueue_kling_16_image_to_video_webhook(args).await?; + + Ok(()) + } +} +*/ \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs b/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs new file mode 100644 index 0000000000..74dead225d --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/webhook/video/mod.rs @@ -0,0 +1 @@ +pub mod enqueue_kling_16_image_to_video_webhook; \ No newline at end of file diff --git a/crates/schema/database/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs b/crates/schema/database/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs index 8d02c6ec49..40825997d8 100644 --- a/crates/schema/database/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs +++ b/crates/schema/database/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs @@ -44,7 +44,7 @@ pub struct MediaFileInsertBuilder { // Media info maybe_mime_type: Option, file_size_bytes: u64, // NB: Non-nullable field - // maybe_duration_millis: Option, + maybe_duration_millis: Option, // maybe_audio_encoding: Option<&'a str>, // maybe_video_encoding: Option<&'a str>, maybe_frame_width: Option, @@ -90,6 +90,7 @@ impl MediaFileInsertBuilder { origin_product_category: MediaFileOriginProductCategory::Unknown, maybe_mime_type: None, file_size_bytes: 0, + maybe_duration_millis: None, maybe_frame_width: None, maybe_frame_height: None, checksum_sha2: None, @@ -173,19 +174,33 @@ impl MediaFileInsertBuilder { self } - // TODO: maybe_duration_millis + pub fn maybe_duration_millis(mut self, maybe_duration_millis: Option) -> Self { + self.maybe_duration_millis = maybe_duration_millis; + self + } + // TODO: maybe_audio_encoding // TODO: maybe_video_encoding - + pub fn frame_width(mut self, frame_width: u32) -> Self { self.maybe_frame_width = Some(frame_width); self } + pub fn maybe_frame_width(mut self, maybe_frame_width: Option) -> Self { + self.maybe_frame_width = maybe_frame_width; + self + } + pub fn frame_height(mut self, frame_height: u32) -> Self { self.maybe_frame_height = Some(frame_height); self } + + pub fn maybe_frame_height(mut self, maybe_frame_height: Option) -> Self { + self.maybe_frame_height = maybe_frame_height; + self + } pub fn checksum_sha2(mut self, checksum_sha2: &str) -> Self { self.checksum_sha2 = Some(checksum_sha2.to_string()); @@ -238,7 +253,7 @@ impl MediaFileInsertBuilder { maybe_origin_filename: None, // TODO maybe_mime_type: self.maybe_mime_type.as_deref(), file_size_bytes: self.file_size_bytes, - maybe_duration_millis: None, // TODO + maybe_duration_millis: self.maybe_duration_millis, maybe_audio_encoding: None, // TODO maybe_video_encoding: None, // TODO maybe_frame_width: self.maybe_frame_width, diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/mod.rs index b1b6b4f003..82a9d3c9e0 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/mod.rs @@ -1 +1,2 @@ -pub mod image; \ No newline at end of file +pub mod image; +pub mod video; \ No newline at end of file diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_image_to_video.rs new file mode 100644 index 0000000000..12049509fb --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_kling_1_6_image_to_video.rs @@ -0,0 +1,25 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +const GENERATE_KLING_1_6_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/kling_16_image_to_video"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateKling16ImageToVideoRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Source of the file to remove the background from. + /// It must be an image. + pub media_file_token: Option, + + /// Optional text prompt. + pub prompt: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateKling16ImageToVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs new file mode 100644 index 0000000000..2c1da8e82a --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs @@ -0,0 +1 @@ +pub mod generate_kling_1_6_image_to_video; \ No newline at end of file diff --git a/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs b/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs index 68df27a89d..36b1deb009 100644 --- a/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs +++ b/crates/schema/public/enums/src/by_table/media_files/media_file_type.rs @@ -111,12 +111,12 @@ impl MediaFileType { match mime_type { "image/jpeg" => Some(Self::Jpg), "image/png" => Some(Self::Png), - //"audio/wav" => Some(Self::Wav), - //"audio/mpeg" => Some(Self::Mp3), - //"image/gif" => Some(Self::Gif), - //"video/mp4" => Some(Self::Mp4), - //"model/gltf-binary" => Some(Self::Glb), - //"model/gltf+json" => Some(Self::Gltf), + "video/mp4" => Some(Self::Mp4), + //"audio/wav" => Some(Self::Wav), // NB: Not sure if this is correct + //"audio/mpeg" => Some(Self::Mp3), // NB: Not sure if this is correct + //"image/gif" => Some(Self::Gif), // NB: Not sure if this is correct + //"model/gltf-binary" => Some(Self::Glb), // NB: Not sure if this is correct + //"model/gltf+json" => Some(Self::Gltf), // NB: Not sure if this is correct _ => None, } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/remove_image_background_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/remove_image_background_handler.rs index ff5ef5ca44..66869a7e17 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/remove_image_background_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/remove_image_background_handler.rs @@ -29,8 +29,8 @@ use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_ use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile}; use tokens::tokens::media_files::MediaFileToken; use utoipa::ToSchema; - - +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; // =============== Error Response =============== @@ -116,6 +116,10 @@ pub async fn remove_image_background_handler( return Err(RemoveImageBackgroundError::BadInput("No media file token provided".to_string())); } }; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(RemoveImageBackgroundError::BadInput(reason)); + } insert_idempotency_token(&request.uuid_idempotency_token, &server_state.mysql_pool) .await @@ -184,10 +188,8 @@ pub async fn remove_image_background_handler( let ip_address = get_request_ip(&http_request); - let uuid_idempotency_token = generate_random_uuid(); - let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { - uuid_idempotency_token: &uuid_idempotency_token, + uuid_idempotency_token: &request.uuid_idempotency_token, maybe_external_third_party_id: &external_job_id, fal_category: FalCategory::BackgroundRemoval, maybe_inference_args: None, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_video_handler.rs new file mode 100644 index 0000000000..63467b824f --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_kling_1_6_video_handler.rs @@ -0,0 +1,225 @@ +use std::fmt; +use std::sync::Arc; + +use crate::http_server::common_requests::media_file_token_path_info::MediaFileTokenPathInfo; +use crate::http_server::common_responses::media::media_links::MediaLinks; +use crate::http_server::deprecated_endpoints::engine::create_scene_handler::CreateSceneError; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::http_server::web_utils::response_success_helpers::simple_json_success; +use crate::state::server_state::ServerState; +use crate::util::delete_role_disambiguation::{delete_role_disambiguation, DeleteRole}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Json; +use actix_web::web::Path; +use actix_web::{web, HttpRequest, HttpResponse}; +use artcraft_api_defs::generate::video::generate_kling_1_6_image_to_video::GenerateKling16ImageToVideoRequest; +use artcraft_api_defs::generate::video::generate_kling_1_6_image_to_video::GenerateKling16ImageToVideoResponse; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::enqueue_kling_16_image_to_video_webhook; +use fal_client::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::Kling16Args; +use fal_client::requests::webhook::video::enqueue_kling_16_image_to_video_webhook::Kling16Duration; +use http_server_common::request::get_request_ip::get_request_ip; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use idempotency::uuid::generate_random_uuid; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +// =============== Error Response =============== + + +#[derive(Debug, Serialize, ToSchema)] +pub enum GenerateKling16VideoError { + BadInput(String), + NotFound, + NotAuthorized, + ServerError, +} + +impl ResponseError for GenerateKling16VideoError { + fn status_code(&self) -> StatusCode { + match *self { + GenerateKling16VideoError::BadInput(_) => StatusCode::BAD_REQUEST, + GenerateKling16VideoError::NotFound => StatusCode::NOT_FOUND, + GenerateKling16VideoError::NotAuthorized => StatusCode::UNAUTHORIZED, + GenerateKling16VideoError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GenerateKling16VideoError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + +/// Background removal +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/kling_16_image_to_video", + responses( + (status = 200, description = "Success", body = RemoveImageBackgroundResponse), + (status = 400, description = "Bad input", body = RemoveImageBackgroundError), + (status = 401, description = "Not authorized", body = RemoveImageBackgroundError), + (status = 500, description = "Server error", body = RemoveImageBackgroundError), + ), + params( + ("request" = RemoveImageBackgroundRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_kling_1_6_video_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, GenerateKling16VideoError> { + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GenerateKling16VideoError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(RemoveImageBackgroundError::NotAuthorized); + // } + //}; + + let media_file_token = match &request.media_file_token { + Some(token) => token, + None => { + warn!("No media file token provided"); + return Err(GenerateKling16VideoError::BadInput("No media file token provided".to_string())); + } + }; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(GenerateKling16VideoError::BadInput(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &server_state.mysql_pool) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + GenerateKling16VideoError::BadInput("invalid idempotency token".to_string()) + })?; + const IS_MOD : bool = false; + + let media_file_lookup_result = get_media_file( + media_file_token, + IS_MOD, + &server_state.mysql_pool, + ).await; + + let media_file = match media_file_lookup_result { + Ok(Some(media_file)) => media_file, + Ok(None) => { + warn!("MediaFile not found: {:?}", media_file_token); + return Err(GenerateKling16VideoError::NotFound); + }, + Err(err) => { + warn!("Error looking up media_file: {:?}", err); + return Err(GenerateKling16VideoError::ServerError); + } + }; + + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(GenerateKling16VideoError::BadInput("Media file must be a JPG or PNG image".to_string())); + } + + let media_domain = get_media_domain(&http_request); + + let bucket_path = MediaFileBucketPath::from_object_hash( + &media_file.public_bucket_directory_hash, + media_file.maybe_public_bucket_prefix.as_deref(), + media_file.maybe_public_bucket_extension.as_deref()); + + let media_links = MediaLinks::from_media_path_and_env( + media_domain, + server_state.server_environment, + &bucket_path); + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let prompt = request.prompt + .as_deref() + .map(|prompt| prompt.trim()) + .unwrap_or_else(|| ""); + + let args = Kling16Args { + image_url: media_links.cdn_url, + webhook_url: &server_state.fal.webhook_url, + duration: Kling16Duration::Default, + prompt: prompt, + api_key: &server_state.fal.api_key, + }; + + let fal_result = enqueue_kling_16_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling remove_background_rembg_webhook: {:?}", err); + GenerateKling16VideoError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + GenerateKling16VideoError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(GenerateKling16VideoError::ServerError); + } + }; + + Ok(Json(GenerateKling16ImageToVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index e69de29bb2..272055b52e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -0,0 +1 @@ +pub mod generate_kling_1_6_video_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs index c77aa37f0c..4768f36687 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal_webhook_handler.rs @@ -2,6 +2,7 @@ use std::fmt; use std::sync::Arc; use crate::http_server::endpoints::webhooks::handle_image_payload::handle_image_payload; +use crate::http_server::endpoints::webhooks::handle_video_payload::handle_video_payload; use crate::state::server_state::ServerState; use actix_web::error::ResponseError; use actix_web::http::StatusCode; @@ -12,9 +13,10 @@ use errors::AnyhowResult; use http_server_common::response::response_success_helpers::SimpleGenericJsonSuccess; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; use log::{info, warn}; +use mysql_queries::queries::generic_inference::web::get_inference_job_by_fal_id::get_inference_job_by_fal_id; use serde_json::Value; use utoipa::ToSchema; -use mysql_queries::queries::generic_inference::web::get_inference_job_by_fal_id::get_inference_job_by_fal_id; + // 1. tauri --> hit endpoint to enqueue // // 2. webhook @@ -141,6 +143,9 @@ pub async fn fal_webhook_handler( if payload_obj.contains_key("image") { handle_image_payload(payload_obj, &job, &server_state).await?; } + if payload_obj.contains_key("video") { + handle_video_payload(payload_obj, &job, &server_state).await?; + } } Ok(SimpleGenericJsonSuccess::wrapped(true)) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs index 86ba6d7df6..53b585b83e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_image_payload.rs @@ -19,7 +19,7 @@ use mysql_queries::queries::media_files::create::insert_builder::media_file_inse use serde_json::{Map, Value}; use std::sync::Arc; -const PREFIX : Option<&str> = Some("image_"); +const PREFIX : Option<&str> = Some("artcraft_"); #[derive(Deserialize, Debug)] pub struct FalWebhookImage { @@ -99,7 +99,7 @@ pub async fn handle_image_payload( .insert_pool(&server_state.mysql_pool) .await?; - info!("Media file uploaded with token: {}", media_token); + info!("Image media file uploaded with token: {}", media_token); Ok(()) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_video_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_video_payload.rs new file mode 100644 index 0000000000..959aa487eb --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/handle_video_payload.rs @@ -0,0 +1,134 @@ +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::state::server_state::ServerState; +use crate::util::http_download_url_to_bytes::http_download_url_to_bytes; +use crate::util::http_download_url_to_tempfile::http_download_url_to_tempfile; +use actix_multipart::form::tempfile::TempFile; +use actix_web::web; +use anyhow::anyhow; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use errors::AnyhowResult; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use images::image_info::image_info::ImageInfo; +use log::{info, warn}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use mysql_queries::queries::generic_inference::web::get_inference_job_by_fal_id::FalJobDetails; +use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +use serde_json::{Map, Value}; +use std::io::Write; +use std::sync::Arc; +use tempfile::NamedTempFile; +use videos::ffprobe_get_info::ffprobe_get_info; + +const PREFIX : Option<&str> = Some("artcraft_"); + +#[derive(Deserialize, Debug)] +pub struct FalWebhookVideo { + pub content_type: Option, + pub file_name: Option, + pub file_size: Option, + pub height: Option, // NB: Might not be on videos + pub width: Option, // NB: Might not be on videos + pub url: Option, +} + +pub async fn handle_video_payload( + payload: &Map, + job: &FalJobDetails, + server_state: &ServerState, +) -> AnyhowResult<()> { + + let video_value = payload.get("video") + .ok_or_else(|| anyhow!("no `image` key in payload"))?; + + info!("Video: {:?}", video_value); + + let video: FalWebhookVideo = serde_json::from_value(video_value.clone())?; + + info!("Video payload received: {:?}", video); + + let video_url = video.url + .as_deref() + .ok_or_else(|| anyhow!("no `url` in video payload"))?; + + //let mime_type = image.content_type + // .as_deref() + // .ok_or_else(|| anyhow!("no `content_type` in image payload"))?; + + let file_bytes = http_download_url_to_bytes(video_url) + .await + .map_err(|e| anyhow!("Failed to download image: {:?}", e))?; + + let mimetype_info = MimetypeInfo::get_for_bytes(&file_bytes) + .ok_or_else(|| anyhow!("Failed to get mimetype info"))?; + + let mime_type = mimetype_info.mime_type(); + + let media_file_type = MediaFileType::try_from_mime_type(mime_type) + .ok_or_else(|| anyhow!("Unsupported media file type: {}", mime_type))?; + + let extension_with_period = mimetype_info.file_extension() + .map(|ext| ext.extension_with_period()) + .ok_or_else(|| anyhow!("Failed to get file extension from mimetype info"))?; + + let file_size_bytes = file_bytes.len(); + let file_hash = sha256_hash_bytes(&file_bytes)?; + + let mut maybe_duration_millis = None; + let mut maybe_frame_width = None; + let mut maybe_frame_height = None; + + if let Ok(mut file) = NamedTempFile::new() { + // NB: Fallible + let _r = file.write_all(&file_bytes); + + if let Ok(video_info) = ffprobe_get_info(&file.path()) { + maybe_duration_millis = video_info.duration + .as_ref() + .map(|duration| duration.millis as u64); + + maybe_frame_width = video_info.dimensions + .as_ref() + .map(|ref dims| dims.width as u32); + + maybe_frame_height = video_info.dimensions + .as_ref() + .map(|dims| dims.height as u32); + } + } + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension_with_period)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type_process( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + &mime_type) + .await?; + + let media_token = MediaFileInsertBuilder::new() + .maybe_creator_user(job.maybe_creator_user_token.as_ref()) + .maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref()) + .creator_ip_address(&job.creator_ip_address) + .public_bucket_directory_hash(&public_upload_path) + .media_file_class(MediaFileClass::Image) + .media_file_type(media_file_type) + .media_file_origin_category(MediaFileOriginCategory::Inference) + //.media_file_origin_product_category(MediaFileOriginProductCategory::Unknown) + .mime_type(mime_type) + .file_size_bytes(file_size_bytes as u64) + .maybe_frame_width(maybe_frame_width) + .maybe_frame_height(maybe_frame_height) + .maybe_duration_millis(maybe_duration_millis) + .checksum_sha2(&file_hash) + .insert_pool(&server_state.mysql_pool) + .await?; + + info!("Video media file uploaded with token: {}", media_token); + + Ok(()) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs index 99c6e1b9d1..21ba1676fe 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/mod.rs @@ -1,2 +1,3 @@ pub mod fal_webhook_handler; -pub mod handle_image_payload; \ No newline at end of file +pub mod handle_image_payload; +pub mod handle_video_payload; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index ab61c30b56..3a9ce5efed 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -3,6 +3,7 @@ use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::{web, App, Error, HttpResponse}; +use crate::http_server::endpoints::generate::video::generate_kling_1_6_video_handler::generate_kling_1_6_video_handler; pub fn add_generate_routes (app: App) -> App where @@ -22,5 +23,11 @@ where .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service(web::scope("/video") + .service(web::resource("/kling_16_image_to_video") + .route(web::post().to(generate_kling_1_6_video_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) }