capture stderr for svc and rvc

_sql/actions/generic_inference_jobs.sql

@@ -19,7 +19,14 @@ set status = 'dead'
 where status IN ('pending', 'started', 'attempt_failed')
 and maybe_model_type IN ('sad_talker', 'so_vits_svc');
 
+-- Tacotron 2 TTS
 update generic_inference_jobs
 set status = 'dead'
 where status IN ('pending', 'started', 'attempt_failed')
   and maybe_model_type IN ('tacotron2');
+
+-- Sad Talker Face Animation
+update generic_inference_jobs
+set status = 'dead'
+where status IN ('pending', 'started', 'attempt_failed')
+  and maybe_model_type IN ('sad_talker');

crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs

@@ -1,3 +1,4 @@
+use std::fs::read_to_string;
 use std::path::PathBuf;
 use std::time::Instant;
 
@@ -220,6 +221,8 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
   //let output_metadata_fs_path = temp_dir.path().join("metadata.json");
   //let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json");
 
+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+
   info!("Running RVC (v2) VC inference...");
 
   info!("Expected output audio filename: {:?}", &output_audio_fs_path);
@@ -260,6 +263,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
         rmvpe_path: &model_dependencies.downloaders.rmvpe.filesystem_path,
         input_path: &input_wav_path,
         output_path: &output_audio_fs_path,
+        stderr_output_file: &stderr_output_file,
         maybe_override_f0_method,
         maybe_f0_up_key,
       });
@@ -270,10 +274,26 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
 
   if !command_exit_status.is_success() {
     error!("Inference failed: {:?}", command_exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+
     safe_delete_temp_file(&input_wav_path);
     safe_delete_temp_file(&output_audio_fs_path);
     safe_delete_temp_directory(&work_temp_dir);
-    return Err(ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)));
+
+    return Err(error);
   }
 
   // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //

crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs

@@ -1,13 +1,14 @@
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use anyhow::anyhow;
 use log::info;
 use once_cell::sync::Lazy;
-use subprocess::{Popen, PopenConfig};
+use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
@@ -80,6 +81,8 @@ pub struct InferenceArgs<P: AsRef<Path>, Q: AsRef<Path>> {
   /// --output_audio_filename: output path of wav file.
   pub output_path: P,
 
+  pub stderr_output_file: P,
+
   /// --f0-method: f0 prediction method
   pub maybe_override_f0_method: Option<FundamentalFrequencyMethodForJob>,
 
@@ -267,6 +270,11 @@ impl RvcV2InferenceCommand {
 
     let mut config = PopenConfig::default();
 
+    info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref());
+
+    let stderr_file = File::create(&args.stderr_output_file)?;
+    config.stderr = Redirection::File(stderr_file);
+
     if !env_vars.is_empty() {
       config.env = Some(env_vars);
     }

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs

@@ -1,3 +1,4 @@
+use std::fs::read_to_string;
 use std::time::Instant;
 
 use anyhow::anyhow;
@@ -148,6 +149,8 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
 
   let output_audio_fs_path = work_temp_dir.path().join("output.wav");
 
+  let stderr_output_file = work_temp_dir.path().join("stderr.txt");
+
   info!("Running VC inference...");
 
   info!("Expected output audio filename: {:?}", &output_audio_fs_path);
@@ -187,6 +190,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
         model_path: &so_vits_svc_fs_path,
         input_path: &input_wav_path,
         output_path: &output_audio_fs_path,
+        stderr_output_file: &stderr_output_file,
         maybe_config_path: None,
         device: Device::Cuda,
         auto_predict_f0,
@@ -199,10 +203,26 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
 
   if !command_exit_status.is_success() {
     error!("Inference failed: {:?}", command_exit_status);
+
+    let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status));
+
+    if let Ok(contents) = read_to_string(&stderr_output_file) {
+      warn!("Captured stderr output: {}", contents);
+
+      //match categorize_error(&contents)  {
+      //  Some(ProcessSingleJobError::FaceDetectionFailure) => {
+      //    warn!("Face not detected in source image");
+      //    error = ProcessSingleJobError::FaceDetectionFailure;
+      //  }
+      //  _ => {}
+      //}
+    }
+
     safe_delete_temp_file(&input_wav_path);
     safe_delete_temp_file(&output_audio_fs_path);
     safe_delete_temp_directory(&work_temp_dir);
-    return Err(ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)));
+
+    return Err(error);
   }
 
   // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs

@@ -1,13 +1,14 @@
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use anyhow::anyhow;
 use log::info;
 use once_cell::sync::Lazy;
-use subprocess::{Popen, PopenConfig};
+use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
@@ -90,6 +91,8 @@ pub struct InferenceArgs<P: AsRef<Path>> {
   /// --output_path: output path of converting model to onnx (which we use to test validity)
   pub output_path: P,
 
+  pub stderr_output_file: P,
+
   /// --auto-predict-f0: turn on or off fundamental frequency auto prediction
   /// This sounds better when left off, but it defaults to *ON* if not specified.
   pub auto_predict_f0: bool,
@@ -366,6 +369,11 @@ impl SoVitsSvcInferenceCommand {
 
     let mut config = PopenConfig::default();
 
+    info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref());
+
+    let stderr_file = File::create(&args.stderr_output_file)?;
+    config.stderr = Redirection::File(stderr_file);
+
     if !env_vars.is_empty() {
       config.env = Some(env_vars);
     }