SadTalker dependencies

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs

@@ -4,5 +4,6 @@ pub mod download_image_file;
 pub mod model_downloaders;
 pub mod process_job;
 pub mod resize_image;
+pub mod sad_talker_dependencies;
 pub mod sad_talker_inference_command;
 pub mod validate_job;

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs

@@ -47,6 +47,13 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess
       .new_generic_inference(job.inference_job_token.as_str())
       .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
 
+  let model_dependencies = args
+      .job_dependencies
+      .job_specific_dependencies
+      .maybe_sad_talker_dependencies
+      .as_ref()
+      .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("missing SadTalker dependencies".to_string())))?;
+
   // ==================== UNPACK + VALIDATE INFERENCE ARGS ==================== //
 
   let job_args = validate_job(job)?;
@@ -57,7 +64,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess
 
   let mut i : usize = 0;
 
-  for downloader in deps.job_type_details.sad_talker.downloaders.all_downloaders() {
+  for downloader in model_dependencies.downloaders.all_downloaders() {
 
     // Temporary debugging
     info!("Downloader {}", i);
@@ -182,9 +189,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess
   let stderr_output_file = work_temp_dir.path().join("stderr.txt");
   let inference_start_time = Instant::now();
 
-  let command_exit_status = args.job_dependencies
-      .job_type_details
-      .sad_talker
+  let command_exit_status = model_dependencies
       .inference_command
       .execute_inference(InferenceArgs {
         input_audio: &audio_path.filesystem_path,
@@ -243,9 +248,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result<JobSuccess
 
     finished_file = output_video_fs_path_watermark.clone();
 
-    let command_exit_status = args.job_dependencies
-        .job_type_details
-        .sad_talker
+    let command_exit_status = model_dependencies
         .ffmpeg_watermark_command
         .execute_inference(ffmpeg_logo_watermark_command::InferenceArgs {
           video_path: &output_video_fs_path,

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_dependencies.rs

@@ -0,0 +1,21 @@
+use errors::AnyhowResult;
+
+use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders;
+use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand;
+use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand;
+
+pub struct SadTalkerDependencies {
+  pub downloaders: SadTalkerDownloaders,
+  pub ffmpeg_watermark_command: FfmpegLogoWatermarkCommand,
+  pub inference_command: SadTalkerInferenceCommand,
+}
+
+impl SadTalkerDependencies {
+  pub fn setup() -> AnyhowResult<Self> {
+    Ok(Self {
+      downloaders: SadTalkerDownloaders::build_all_from_env(),
+      ffmpeg_watermark_command: FfmpegLogoWatermarkCommand::from_env()?,
+      inference_command: SadTalkerInferenceCommand::from_env()?,
+    })
+  }
+}

crates/service/job/inference_job/src/job_dependencies.rs

@@ -24,15 +24,12 @@ use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improve
 use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference;
 use newrelic_telemetry::Client as NewRelicClient;
 
-use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders;
-use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand;
 use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders;
 use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand;
 use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand;
 use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand;
 use crate::job_specific_dependencies::JobSpecificDependencies;
-use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand;
 use crate::util::scoped_execution::ScopedExecution;
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
 
@@ -163,7 +160,6 @@ pub struct JobCaches {
 pub struct JobTypeDetails {
   pub tacotron2_old_vocodes: Tacotron2VocodesDetails,
   pub vits: VitsDetails,
-  pub sad_talker: SadTalkerDetails,
   pub vall_e_x:  VallEXDetails
   //pub tacotron2_modern: ...,
   //pub softvc: ...,
@@ -187,13 +183,7 @@ pub struct VitsDetails {
   pub inference_command: VitsInferenceCommand,
 }
 
-pub struct SadTalkerDetails {
-  pub downloaders: SadTalkerDownloaders,
-  pub inference_command: SadTalkerInferenceCommand,
-  pub ffmpeg_watermark_command: FfmpegLogoWatermarkCommand,
-}
-
-// TODO: we will probably want a command type of some kind that implements 
+// TODO: we will probably want a command type of some kind that implements
 // some kind of interface that we can just pass strings to.
 pub struct VallEXDetails {
   pub downloaders: VallEXDownloaders,

crates/service/job/inference_job/src/job_specific_dependencies.rs

@@ -3,12 +3,14 @@ use log::info;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use errors::AnyhowResult;
 
+use crate::job::job_types::lipsync::sad_talker::sad_talker_dependencies::SadTalkerDependencies;
 use crate::job::job_types::vc::rvc_v2::rvc_v2_dependencies::RvcV2Dependencies;
 use crate::job::job_types::vc::so_vits_svc::svc_dependencies::SvcDependencies;
 use crate::util::scoped_execution::ScopedExecution;
 
 pub struct JobSpecificDependencies {
   pub maybe_rvc_v2_dependencies: Option<RvcV2Dependencies>,
+  pub maybe_sad_talker_dependencies: Option<SadTalkerDependencies>,
   pub maybe_svc_dependencies: Option<SvcDependencies>,
 }
 
@@ -16,6 +18,7 @@ impl JobSpecificDependencies {
 
   pub fn setup_for_jobs(scoped_execution: &ScopedExecution) -> AnyhowResult<Self> {
     let mut maybe_rvc_v2_dependencies = None;
+    let mut maybe_sad_talker_dependencies = None;
     let mut maybe_svc_dependencies = None;
 
     if scoped_execution.can_run_job(InferenceModelType::RvcV2) {
@@ -23,6 +26,11 @@ impl JobSpecificDependencies {
       maybe_rvc_v2_dependencies = Some(RvcV2Dependencies::setup()?);
     }
 
+    if scoped_execution.can_run_job(InferenceModelType::SadTalker) {
+      info!("Setting SadTalker dependencies...");
+      maybe_sad_talker_dependencies = Some(SadTalkerDependencies::setup()?);
+    }
+
     if scoped_execution.can_run_job(InferenceModelType::SoVitsSvc) {
       info!("Setting SVC dependencies...");
       maybe_svc_dependencies = Some(SvcDependencies::setup()?);
@@ -30,6 +38,7 @@ impl JobSpecificDependencies {
 
     Ok(JobSpecificDependencies {
       maybe_rvc_v2_dependencies,
+      maybe_sad_talker_dependencies,
       maybe_svc_dependencies
     })
   }

crates/service/job/inference_job/src/main.rs

@@ -50,16 +50,13 @@ use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, Doc
 use crate::http_server::run_http_server::CreateServerArgs;
 use crate::http_server::run_http_server::launch_http_server;
 use crate::job::job_loop::main_loop::main_loop;
-use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders;
-use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand;
 use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders;
 use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand;
 use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand;
 use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand;
-use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, SadTalkerDetails, Tacotron2VocodesDetails, VallEXDetails, VitsDetails};
+use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, Tacotron2VocodesDetails, VallEXDetails, VitsDetails};
 use crate::job_specific_dependencies::JobSpecificDependencies;
-use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand;
 use crate::util::scoped_execution::ScopedExecution;
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
 
@@ -322,11 +319,6 @@ async fn main() -> AnyhowResult<()> {
       vits: VitsDetails {
         inference_command: vits_inference_command()?,
       },
-      sad_talker: SadTalkerDetails {
-        downloaders: SadTalkerDownloaders::build_all_from_env(),
-        inference_command: SadTalkerInferenceCommand::from_env()?,
-        ffmpeg_watermark_command: FfmpegLogoWatermarkCommand::from_env()?,
-      },
     },
     container: container_environment.clone(),
     container_db: ContainerEnvironmentArg {