diff --git a/Cargo.lock b/Cargo.lock index 882400c47d..128a09f294 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6865,6 +6865,7 @@ dependencies = [ "strum", "strum_macros", "symphonia", + "thumbnail_generator", "time", "tokens", "tokio", diff --git a/crates/lib/api_clients/thumbnail_generator/src/task_client/thumbnail_task.rs b/crates/lib/api_clients/thumbnail_generator/src/task_client/thumbnail_task.rs index 48040abe2b..7a0f97ac31 100644 --- a/crates/lib/api_clients/thumbnail_generator/src/task_client/thumbnail_task.rs +++ b/crates/lib/api_clients/thumbnail_generator/src/task_client/thumbnail_task.rs @@ -1,6 +1,62 @@ use log::error; use serde_json::{json, Value}; + +#[derive(Debug)] +pub enum ThumbnailTaskInputMimeType { + MP4 +} + +pub enum ThumbnailTaskOutputMimeType { + PNG, + JPEG, + GIF, +} + +impl ThumbnailTaskInputMimeType { + fn as_str(&self) -> &str { + match self { + ThumbnailTaskInputMimeType::MP4 => "video/mp4", + } + } + + fn as_string(&self) -> String { + self.as_str().to_string() + } + + fn to_output_mimetypes(&self) -> Vec { + match self { + ThumbnailTaskInputMimeType::MP4 => vec![ + ThumbnailTaskOutputMimeType::JPEG, + ThumbnailTaskOutputMimeType::GIF, + ], + } + } +} + +impl ThumbnailTaskOutputMimeType { + fn as_str(&self) -> &str { + match self { + ThumbnailTaskOutputMimeType::PNG => "image/png", + ThumbnailTaskOutputMimeType::JPEG => "image/jpeg", + ThumbnailTaskOutputMimeType::GIF => "image/gif", + } + } + + fn as_string(&self) -> String { + self.as_str().to_string() + } + + fn to_extension(&self) -> &str { + match self { + ThumbnailTaskOutputMimeType::PNG => "png", + ThumbnailTaskOutputMimeType::JPEG => "jpg", + ThumbnailTaskOutputMimeType::GIF => "gif", + } + } +} + + #[derive(Debug)] pub enum ThumbnailTaskError { InvalidTask, @@ -19,22 +75,20 @@ struct ThumbnailTask{ #[derive(Debug)] pub struct ThumbnailTaskBuilder{ + source_mimetype: ThumbnailTaskInputMimeType, bucket: Option, path: Option, - source_mimetype: Option, - output_mimetype: Option, output_extension: Option, output_suffix: Option, event_id: Option, } impl ThumbnailTaskBuilder { - pub fn new() -> Self { + pub fn new_for_source_mimetype(source_mimetype: ThumbnailTaskInputMimeType) -> Self { Self { + source_mimetype, bucket: None, path: None, - source_mimetype: None, - output_mimetype: None, output_extension: None, output_suffix: None, event_id: None, @@ -51,16 +105,6 @@ impl ThumbnailTaskBuilder { self } - pub fn with_source_mimetype(&mut self, source_mimetype: &str) -> &mut Self { - self.source_mimetype = Some(source_mimetype.to_string()); - self - } - - pub fn with_output_mimetype(&mut self, output_mimetype: &str) -> &mut Self { - self.output_mimetype = Some(output_mimetype.to_string()); - self - } - pub fn with_output_extension(&mut self, output_extension: &str) -> &mut Self { self.output_extension = Some(output_extension.to_string()); self @@ -76,30 +120,43 @@ impl ThumbnailTaskBuilder { self } - fn build(&self) -> Result { + fn build(&self, thumbnail_task_output_mime_type: ThumbnailTaskOutputMimeType) -> Result { Ok(ThumbnailTask { + source_mimetype: self.source_mimetype.as_string(), + output_mimetype: thumbnail_task_output_mime_type.as_string(), bucket: self.bucket.clone().ok_or("bucket is required")?, path: self.path.clone().ok_or("path is required")?, - source_mimetype: self.source_mimetype.clone().ok_or("source_mimetype is required")?, - output_mimetype: self.output_mimetype.clone().ok_or("output_mimetype is required")?, output_extension: self.output_extension.clone().ok_or("output_extension is required")?, output_suffix: self.output_suffix.clone().unwrap_or("".to_string()), event_id: self.event_id.clone().ok_or("event_id is required")?, }) } - pub async fn send(&self) -> Result<(), ThumbnailTaskError> { - match self.build() { - Ok(task) => { - match task.send().await { - Ok(_) => Ok(()), - Err(_) => Err(ThumbnailTaskError::RequestError), + pub async fn send_all(&self) -> Result<(), ThumbnailTaskError> { + let mut results: Vec> = vec![]; + for output_mimetype in self.source_mimetype.to_output_mimetypes() { + match self.build(output_mimetype) { + Ok(task) => { + match task.send().await { + Ok(_) => {}, + Err(_) => { + error!("Failed to send thumbnail task: {:?}", self); + results.push(Err(ThumbnailTaskError::RequestError)); + }, + } } - } - Err(_) => { - error!("Invalid thumbnail task: {:?}", self); - Err(ThumbnailTaskError::InvalidTask) - }, + Err(_) => { + error!("Invalid thumbnail task: {:?}", self); + results.push(Err(ThumbnailTaskError::InvalidTask)); + }, + }; + + } + + if results.iter().any(|r| r.is_err()) { + Err(ThumbnailTaskError::RequestError) + } else { + Ok(()) } } } diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 79d5427c21..a6473613a9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -15,7 +15,7 @@ use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_file::get_mimetype_for_file; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_files::create::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; -use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder; +use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType}; use tokens::tokens::users::UserToken; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; @@ -320,37 +320,21 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result vec!["image/gif", "image/jpeg"], - _ => vec![], - }; - for output_type in thumbnail_types { - match get_output_file_extension_from_mimetype(output_type) { - Ok(output_extension) => { - let thumbnail_task_result = ThumbnailTaskBuilder::new() - .with_bucket(&*args.job_dependencies.buckets.public_bucket_client.bucket_name()) - .with_path(&*path_to_string(result_bucket_object_pathbuf.clone())) - .with_source_mimetype(mimetype.as_str()) - .with_output_mimetype(output_type) - .with_output_suffix("thumb") - .with_output_extension(output_extension) - .with_event_id(&job.id.0.to_string()) - .send() - .await; + let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4) + .with_bucket(&*args.job_dependencies.buckets.public_bucket_client.bucket_name()) + .with_path(&*path_to_string(result_bucket_object_pathbuf.clone())) + .with_output_suffix("thumb") + .with_event_id(&job.id.0.to_string()) + .send_all() + .await; - match thumbnail_task_result { - Ok(thumbnail_task) => { - debug!("Thumbnail task created: {:?}", thumbnail_task); - }, - Err(e) => { - error!("Failed to create thumbnail task: {:?}", e); - } - } - }, - Err(e) => { - error!("Failed to get output file extension from mimetype: {:?}", e); - } + match thumbnail_task_result { + Ok(thumbnail_task) => { + debug!("Thumbnail tasks sent: {:?}", thumbnail_task); + }, + Err(e) => { + error!("Failed to create some/all thumbnail tasks: {:?}", e); } } // ==================== DELETE TEMP FILES ==================== // diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/live_portrait/process_live_portrait_job.rs b/crates/service/job/inference_job/src/job/job_types/workflow/live_portrait/process_live_portrait_job.rs index 8a0116b054..0dd10616a9 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/live_portrait/process_live_portrait_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/live_portrait/process_live_portrait_job.rs @@ -36,7 +36,7 @@ use mysql_queries::queries::media_files::get::get_media_file::get_media_file; use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token; use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection}; -use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder; +use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType}; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::prompts::PromptToken; use videos::ffprobe_get_dimensions::ffprobe_get_dimensions; @@ -304,6 +304,7 @@ pub async fn process_live_portrait_job( ProcessSingleJobError::Other(e) })?; + // ==================== (OPTIONAL) DEBUG SLEEP ==================== // if let Some(sleep_millis) = job_payload.sleep_millis { @@ -311,6 +312,25 @@ pub async fn process_live_portrait_job( thread::sleep(Duration::from_millis(sleep_millis)); } + // ==================== GENERATE THUMBNAILS ==================== // + + let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4) + .with_bucket(&*deps.buckets.public_bucket_client.bucket_name()) + .with_path(&*path_to_string(result_bucket_object_pathbuf.clone())) + .with_output_suffix("thumb") + .with_event_id(&job.id.0.to_string()) + .send_all() + .await; + + match thumbnail_task_result { + Ok(thumbnail_task) => { + debug!("Thumbnail tasks sent: {:?}", thumbnail_task); + }, + Err(e) => { + error!("Failed to create some/all thumbnail tasks: {:?}", e); + } + } + // ==================== CLEANUP/ DELETE TEMP FILES ==================== // info!("Cleaning up temporary files..."); diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/steps/validate_and_save_results.rs b/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/steps/validate_and_save_results.rs index 33a9d454d8..4aea27478a 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/steps/validate_and_save_results.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/steps/validate_and_save_results.rs @@ -24,7 +24,7 @@ use mysql_queries::payloads::prompt_args::prompt_inner_payload::PromptInnerPaylo use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_files::create::insert_media_file_from_comfy_ui::{insert_media_file_from_comfy_ui, InsertArgs}; use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; -use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder; +use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType}; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::prompts::PromptToken; @@ -130,40 +130,20 @@ pub async fn validate_and_save_results(args: SaveResultsArgs<'_>) -> Result vec!["image/gif", "image/jpeg"], - _ => vec![], - }; + let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4) + .with_bucket(&*args.deps.buckets.public_bucket_client.bucket_name()) + .with_path(&*path_to_string(result_bucket_object_pathbuf.clone())) + .with_output_suffix("thumb") + .with_event_id(&args.job.id.0.to_string()) + .send_all() + .await; - info!("Generating thumbnail tasks..."); - - for output_type in thumbnail_types { - let output_ext = match get_file_extension(output_type) { - Ok(ext) => format!("{}", ext), - Err(e) => { - error!("Failed to get extension for output type: {:?}", e); - continue; - } - }; - let thumbnail_task_result = ThumbnailTaskBuilder::new() - .with_bucket(&*args.deps.buckets.public_bucket_client.bucket_name()) - .with_path(&*path_to_string(result_bucket_object_pathbuf.clone())) - .with_source_mimetype(mimetype.as_str()) - .with_output_mimetype(output_type) - .with_output_suffix("thumb") - .with_output_extension(&output_ext) - .with_event_id(&args.job.id.0.to_string()) - .send() - .await; - - match thumbnail_task_result { - Ok(thumbnail_task) => { - debug!("Thumbnail task created: {:?}", thumbnail_task); - }, - Err(e) => { - error!("Failed to create thumbnail task: {:?}", e); - } + match thumbnail_task_result { + Ok(thumbnail_task) => { + debug!("Thumbnail tasks sent: {:?}", thumbnail_task); + }, + Err(e) => { + error!("Failed to create some/all thumbnail tasks: {:?}", e); } } diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 0b4dd68c12..7015c20765 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -63,6 +63,7 @@ redis_common = { path = "../../../lib/redis_common" } reusable_types = { path = "../../../_types/reusable_types" } server_environment = { path = "../../../lib/server_environment" } tokens = { path = "../../../_types/tokens" } +thumbnail_generator = { path = "../../../lib/api_clients/thumbnail_generator" } tts_common = { path = "../../../lib/tts_common" } url_config = { path = "../../../lib/url_config" } user_input_common = { path = "../../../lib/user_input_common" } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_video_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_video_media_file_handler.rs index 550a6650ec..fc71c81f18 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_video_media_file_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_video_media_file_handler.rs @@ -8,7 +8,7 @@ use actix_multipart::form::tempfile::TempFile; use actix_multipart::form::text::Text; use actix_multipart::Multipart; use actix_web::{HttpRequest, HttpResponse, web}; -use log::{error, info, warn}; +use log::{debug, error, info, warn}; use once_cell::sync::Lazy; use stripe::CreatePaymentLinkShippingAddressCollectionAllowedCountries::Mf; use utoipa::ToSchema; @@ -20,6 +20,7 @@ use enums::by_table::media_files::media_file_subtype::MediaFileSubtype; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; +use filesys::path_to_string::path_to_string; use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; use http_server_common::request::get_request_ip::get_request_ip; use media::decode_basic_audio_info::decode_basic_audio_bytes_info; @@ -30,6 +31,7 @@ use mysql_queries::queries::media_files::create::insert_media_file_from_file_upl use mysql_queries::queries::media_files::get::get_media_file::get_media_file; use tokens::tokens::media_files::MediaFileToken; use videos::get_mp4_info::{get_mp4_info, get_mp4_info_for_bytes, get_mp4_info_for_bytes_and_len}; +use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType}; use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; use crate::state::server_state::ServerState; @@ -319,6 +321,23 @@ pub async fn upload_new_video_media_file_handler( info!("new media file id: {} token: {:?}", record_id, &token); + let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4) + .with_bucket(server_state.public_bucket_client.bucket_name().as_str()) + .with_path(&*path_to_string(public_upload_path.to_full_object_pathbuf())) + .with_output_suffix("thumb") + .with_event_id(&token.to_string()) + .send_all() + .await; + + match thumbnail_task_result { + Ok(thumbnail_task) => { + debug!("Thumbnail tasks sent: {:?}", thumbnail_task); + }, + Err(e) => { + error!("Failed to create some/all thumbnail tasks: {:?}", e); + } + } + let response = UploadNewVideoMediaFileSuccessResponse { success: true, media_file_token: token,