make making thumbnails easier (#358)

Cargo.lock

@@ -6865,6 +6865,7 @@ dependencies = [
  "strum",
  "strum_macros",
  "symphonia",
+ "thumbnail_generator",
  "time",
  "tokens",
  "tokio",

crates/lib/api_clients/thumbnail_generator/src/task_client/thumbnail_task.rs

@@ -1,6 +1,62 @@
 use log::error;
 use serde_json::{json, Value};
 
+
+#[derive(Debug)]
+pub enum ThumbnailTaskInputMimeType {
+    MP4
+}
+
+pub enum ThumbnailTaskOutputMimeType {
+    PNG,
+    JPEG,
+    GIF,
+}
+
+impl ThumbnailTaskInputMimeType {
+    fn as_str(&self) -> &str {
+        match self {
+            ThumbnailTaskInputMimeType::MP4 => "video/mp4",
+        }
+    }
+
+    fn as_string(&self) -> String {
+        self.as_str().to_string()
+    }
+
+    fn to_output_mimetypes(&self) -> Vec<ThumbnailTaskOutputMimeType> {
+        match self {
+            ThumbnailTaskInputMimeType::MP4 => vec![
+                ThumbnailTaskOutputMimeType::JPEG,
+                ThumbnailTaskOutputMimeType::GIF,
+            ],
+        }
+    }
+}
+
+impl ThumbnailTaskOutputMimeType {
+    fn as_str(&self) -> &str {
+        match self {
+            ThumbnailTaskOutputMimeType::PNG => "image/png",
+            ThumbnailTaskOutputMimeType::JPEG => "image/jpeg",
+            ThumbnailTaskOutputMimeType::GIF => "image/gif",
+        }
+    }
+
+    fn as_string(&self) -> String {
+        self.as_str().to_string()
+    }
+
+    fn to_extension(&self) -> &str {
+        match self {
+            ThumbnailTaskOutputMimeType::PNG => "png",
+            ThumbnailTaskOutputMimeType::JPEG => "jpg",
+            ThumbnailTaskOutputMimeType::GIF => "gif",
+        }
+    }
+}
+
+
 #[derive(Debug)]
 pub enum ThumbnailTaskError {
     InvalidTask,
@@ -19,22 +75,20 @@ struct ThumbnailTask{
 
 #[derive(Debug)]
 pub struct ThumbnailTaskBuilder{
+    source_mimetype: ThumbnailTaskInputMimeType,
     bucket: Option<String>,
     path: Option<String>,
-    source_mimetype: Option<String>,
-    output_mimetype: Option<String>,
     output_extension: Option<String>,
     output_suffix: Option<String>,
     event_id: Option<String>,
 }
 
 impl ThumbnailTaskBuilder {
-    pub fn new() -> Self {
+    pub fn new_for_source_mimetype(source_mimetype: ThumbnailTaskInputMimeType) -> Self {
         Self {
+            source_mimetype,
             bucket: None,
             path: None,
-            source_mimetype: None,
-            output_mimetype: None,
             output_extension: None,
             output_suffix: None,
             event_id: None,
@@ -51,16 +105,6 @@ impl ThumbnailTaskBuilder {
         self
     }
 
-    pub fn with_source_mimetype(&mut self, source_mimetype: &str) -> &mut Self {
-        self.source_mimetype = Some(source_mimetype.to_string());
-        self
-    }
-
-    pub fn with_output_mimetype(&mut self, output_mimetype: &str) -> &mut Self {
-        self.output_mimetype = Some(output_mimetype.to_string());
-        self
-    }
-
     pub fn with_output_extension(&mut self, output_extension: &str) -> &mut Self {
         self.output_extension = Some(output_extension.to_string());
         self
@@ -76,30 +120,43 @@ impl ThumbnailTaskBuilder {
         self
     }
 
-    fn build(&self) -> Result<ThumbnailTask, String> {
+    fn build(&self, thumbnail_task_output_mime_type: ThumbnailTaskOutputMimeType) -> Result<ThumbnailTask, String> {
         Ok(ThumbnailTask {
+            source_mimetype: self.source_mimetype.as_string(),
+            output_mimetype: thumbnail_task_output_mime_type.as_string(),
             bucket: self.bucket.clone().ok_or("bucket is required")?,
             path: self.path.clone().ok_or("path is required")?,
-            source_mimetype: self.source_mimetype.clone().ok_or("source_mimetype is required")?,
-            output_mimetype: self.output_mimetype.clone().ok_or("output_mimetype is required")?,
             output_extension: self.output_extension.clone().ok_or("output_extension is required")?,
             output_suffix: self.output_suffix.clone().unwrap_or("".to_string()),
             event_id: self.event_id.clone().ok_or("event_id is required")?,
         })
     }
 
-    pub async fn send(&self) -> Result<(), ThumbnailTaskError> {
-        match self.build() {
-            Ok(task) => {
-                match task.send().await {
-                    Ok(_) => Ok(()),
-                    Err(_) => Err(ThumbnailTaskError::RequestError),
+    pub async fn send_all(&self) -> Result<(), ThumbnailTaskError> {
+        let mut results: Vec<Result<(), ThumbnailTaskError>> = vec![];
+        for output_mimetype in self.source_mimetype.to_output_mimetypes() {
+            match self.build(output_mimetype) {
+                Ok(task) => {
+                    match task.send().await {
+                        Ok(_) => {},
+                        Err(_) => {
+                            error!("Failed to send thumbnail task: {:?}", self);
+                            results.push(Err(ThumbnailTaskError::RequestError));
+                        },
+                    }
                 }
-            }
-            Err(_) => {
-                error!("Invalid thumbnail task: {:?}", self);
-                Err(ThumbnailTaskError::InvalidTask)
-            },
+                Err(_) => {
+                    error!("Invalid thumbnail task: {:?}", self);
+                    results.push(Err(ThumbnailTaskError::InvalidTask));
+                },
+            };
+
+        }
+
+        if results.iter().any(|r| r.is_err()) {
+            Err(ThumbnailTaskError::RequestError)
+        } else {
+            Ok(())
         }
     }
 }

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs

@@ -15,7 +15,7 @@ use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::media_files::create::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs};
-use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder;
+use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
 use tokens::tokens::users::UserToken;
 
 use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -320,37 +320,21 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess
       .await
       .map_err(|e| ProcessSingleJobError::Other(e))?;
 
-  let thumbnail_types = match mimetype.as_str() {
-    "video/mp4" => vec!["image/gif", "image/jpeg"],
-    _ => vec![],
-  };
 
-  for output_type in thumbnail_types {
-    match get_output_file_extension_from_mimetype(output_type) {
-      Ok(output_extension) => {
-        let thumbnail_task_result = ThumbnailTaskBuilder::new()
-            .with_bucket(&*args.job_dependencies.buckets.public_bucket_client.bucket_name())
-            .with_path(&*path_to_string(result_bucket_object_pathbuf.clone()))
-            .with_source_mimetype(mimetype.as_str())
-            .with_output_mimetype(output_type)
-            .with_output_suffix("thumb")
-            .with_output_extension(output_extension)
-            .with_event_id(&job.id.0.to_string())
-            .send()
-            .await;
+  let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4)
+      .with_bucket(&*args.job_dependencies.buckets.public_bucket_client.bucket_name())
+      .with_path(&*path_to_string(result_bucket_object_pathbuf.clone()))
+      .with_output_suffix("thumb")
+      .with_event_id(&job.id.0.to_string())
+      .send_all()
+      .await;
 
-        match thumbnail_task_result {
-          Ok(thumbnail_task) => {
-            debug!("Thumbnail task created: {:?}", thumbnail_task);
-          },
-          Err(e) => {
-            error!("Failed to create thumbnail task: {:?}", e);
-          }
-        }
-      },
-      Err(e) => {
-          error!("Failed to get output file extension from mimetype: {:?}", e);
-      }
+  match thumbnail_task_result {
+    Ok(thumbnail_task) => {
+      debug!("Thumbnail tasks sent: {:?}", thumbnail_task);
+    },
+    Err(e) => {
+      error!("Failed to create some/all thumbnail tasks: {:?}", e);
     }
   }
   // ==================== DELETE TEMP FILES ==================== //

crates/service/job/inference_job/src/job/job_types/workflow/live_portrait/process_live_portrait_job.rs

@@ -36,7 +36,7 @@ use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
 use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
 use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
 use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
-use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder;
+use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
 use tokens::tokens::media_files::MediaFileToken;
 use tokens::tokens::prompts::PromptToken;
 use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
@@ -304,6 +304,7 @@ pub async fn process_live_portrait_job(
         ProcessSingleJobError::Other(e)
       })?;
 
+
   // ==================== (OPTIONAL) DEBUG SLEEP ==================== //
 
   if let Some(sleep_millis) = job_payload.sleep_millis {
@@ -311,6 +312,25 @@ pub async fn process_live_portrait_job(
     thread::sleep(Duration::from_millis(sleep_millis));
   }
 
+  // ==================== GENERATE THUMBNAILS ==================== //
+
+  let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4)
+    .with_bucket(&*deps.buckets.public_bucket_client.bucket_name())
+    .with_path(&*path_to_string(result_bucket_object_pathbuf.clone()))
+    .with_output_suffix("thumb")
+    .with_event_id(&job.id.0.to_string())
+    .send_all()
+    .await;
+
+  match thumbnail_task_result {
+    Ok(thumbnail_task) => {
+      debug!("Thumbnail tasks sent: {:?}", thumbnail_task);
+    },
+    Err(e) => {
+      error!("Failed to create some/all thumbnail tasks: {:?}", e);
+    }
+  }
+
   // ==================== CLEANUP/ DELETE TEMP FILES ==================== //
 
   info!("Cleaning up temporary files...");

crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/steps/validate_and_save_results.rs

@@ -24,7 +24,7 @@ use mysql_queries::payloads::prompt_args::prompt_inner_payload::PromptInnerPaylo
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::media_files::create::insert_media_file_from_comfy_ui::{insert_media_file_from_comfy_ui, InsertArgs};
 use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
-use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder;
+use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
 use tokens::tokens::media_files::MediaFileToken;
 use tokens::tokens::prompts::PromptToken;
 
@@ -130,40 +130,20 @@ pub async fn validate_and_save_results(args: SaveResultsArgs<'_>) -> Result<Medi
       .await
       .map_err(|e| ProcessSingleJobError::Other(e))?;
 
-  // generate thumbnail using thumbnail service
-  let thumbnail_types = match mimetype.as_str() {
-    "video/mp4" => vec!["image/gif", "image/jpeg"],
-    _ => vec![],
-  };
+  let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4)
+      .with_bucket(&*args.deps.buckets.public_bucket_client.bucket_name())
+      .with_path(&*path_to_string(result_bucket_object_pathbuf.clone()))
+      .with_output_suffix("thumb")
+      .with_event_id(&args.job.id.0.to_string())
+      .send_all()
+      .await;
 
-  info!("Generating thumbnail tasks...");
-
-  for output_type in thumbnail_types {
-    let output_ext = match get_file_extension(output_type) {
-        Ok(ext) => format!("{}", ext),
-        Err(e) => {
-            error!("Failed to get extension for output type: {:?}", e);
-            continue;
-        }
-    };
-    let thumbnail_task_result = ThumbnailTaskBuilder::new()
-        .with_bucket(&*args.deps.buckets.public_bucket_client.bucket_name())
-        .with_path(&*path_to_string(result_bucket_object_pathbuf.clone()))
-        .with_source_mimetype(mimetype.as_str())
-        .with_output_mimetype(output_type)
-        .with_output_suffix("thumb")
-        .with_output_extension(&output_ext)
-        .with_event_id(&args.job.id.0.to_string())
-        .send()
-        .await;
-
-    match thumbnail_task_result {
-      Ok(thumbnail_task) => {
-        debug!("Thumbnail task created: {:?}", thumbnail_task);
-      },
-      Err(e) => {
-        error!("Failed to create thumbnail task: {:?}", e);
-      }
+  match thumbnail_task_result {
+    Ok(thumbnail_task) => {
+      debug!("Thumbnail tasks sent: {:?}", thumbnail_task);
+    },
+    Err(e) => {
+      error!("Failed to create some/all thumbnail tasks: {:?}", e);
     }
   }
 

crates/service/web/storyteller_web/Cargo.toml

@@ -63,6 +63,7 @@ redis_common = { path = "../../../lib/redis_common" }
 reusable_types = { path = "../../../_types/reusable_types" }
 server_environment = { path = "../../../lib/server_environment" }
 tokens = { path = "../../../_types/tokens" }
+thumbnail_generator = { path = "../../../lib/api_clients/thumbnail_generator" }
 tts_common = { path = "../../../lib/tts_common" }
 url_config = { path = "../../../lib/url_config" }
 user_input_common = { path = "../../../lib/user_input_common" }

crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_video_media_file_handler.rs

@@ -8,7 +8,7 @@ use actix_multipart::form::tempfile::TempFile;
 use actix_multipart::form::text::Text;
 use actix_multipart::Multipart;
 use actix_web::{HttpRequest, HttpResponse, web};
-use log::{error, info, warn};
+use log::{debug, error, info, warn};
 use once_cell::sync::Lazy;
 use stripe::CreatePaymentLinkShippingAddressCollectionAllowedCountries::Mf;
 use utoipa::ToSchema;
@@ -20,6 +20,7 @@ use enums::by_table::media_files::media_file_subtype::MediaFileSubtype;
 use enums::by_table::media_files::media_file_type::MediaFileType;
 use enums::common::visibility::Visibility;
 use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
+use filesys::path_to_string::path_to_string;
 use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
 use http_server_common::request::get_request_ip::get_request_ip;
 use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
@@ -30,6 +31,7 @@ use mysql_queries::queries::media_files::create::insert_media_file_from_file_upl
 use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
 use tokens::tokens::media_files::MediaFileToken;
 use videos::get_mp4_info::{get_mp4_info, get_mp4_info_for_bytes, get_mp4_info_for_bytes_and_len};
+use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
 
 use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
 use crate::state::server_state::ServerState;
@@ -319,6 +321,23 @@ pub async fn upload_new_video_media_file_handler(
 
   info!("new media file id: {} token: {:?}", record_id, &token);
 
+  let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4)
+    .with_bucket(server_state.public_bucket_client.bucket_name().as_str())
+    .with_path(&*path_to_string(public_upload_path.to_full_object_pathbuf()))
+    .with_output_suffix("thumb")
+    .with_event_id(&token.to_string())
+    .send_all()
+    .await;
+
+  match thumbnail_task_result {
+    Ok(thumbnail_task) => {
+      debug!("Thumbnail tasks sent: {:?}", thumbnail_task);
+    },
+    Err(e) => {
+      error!("Failed to create some/all thumbnail tasks: {:?}", e);
+    }
+  }
+
   let response = UploadNewVideoMediaFileSuccessResponse {
     success: true,
     media_file_token: token,