diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index d0271657c4..f99d376c74 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -135,6 +135,8 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec (true, false, "not yet implemented".to_string(), Some("not yet implemented")), + ProcessSingleJobError::FaceDetectionFailure => + (true, false, "face not detected".to_string(), Some("face not detected")), // Non-permanent failures ProcessSingleJobError::FilesystemFull => @@ -177,6 +179,7 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec {} ProcessSingleJobError::KeepAliveElapsed => {} ProcessSingleJobError::NotYetImplemented => {} + ProcessSingleJobError::FaceDetectionFailure => {} } } } diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs index 5cc24ad7a7..c6509c1dde 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs @@ -3,6 +3,11 @@ use anyhow::anyhow; /// Error from processing a single job #[derive(Debug)] pub enum ProcessSingleJobError { + /// For inference jobs that are required to find faces, fail the job + /// immediately if no face is found. We can also surface the error to + /// the user in a friendly way. + /// Example job types: SadTalker, Wav2Lip. + FaceDetectionFailure, /// The filesystem is out of space and we need to free it up. FilesystemFull, /// The job is invalid (bad state, etc.) diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 4c112ef3a9..c88a8b2279 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -1,7 +1,8 @@ +use std::fs::read_to_string; use std::time::Instant; use anyhow::anyhow; -use log::{error, info}; +use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; use container_common::filesystem::check_file_exists::check_file_exists; @@ -132,6 +133,8 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result) -> Result) -> Result> { /// --result_file: path to final file output pub output_file: P, + pub stderr_output_file: P, + // TODO: Other SadTalker args } @@ -243,6 +246,11 @@ impl SadTalkerInferenceCommand { let mut config = PopenConfig::default(); + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + if !env_vars.is_empty() { config.env = Some(env_vars); }