From 4e20f63f8717e5a0deb5eca7509ff1b620fe7521 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Feb 2024 23:28:43 -0500 Subject: [PATCH] output stderr on failure --- .../image_generation/sd/process_job.rs | 95 ++++++++++++++++--- .../sd/sd_inference_command.rs | 10 +- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job.rs b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job.rs index 8498767dd5..d83aa5695d 100644 --- a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job.rs @@ -1,9 +1,10 @@ +use std::fs::read_to_string; use std::path::PathBuf; use std::time::{Duration, Instant}; use actix_web::dev::ResourcePath; use anyhow::anyhow; -use log::{error, info}; +use log::{error, info, warn}; use serde_json; use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor; @@ -21,6 +22,8 @@ use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::by_table::model_weights::weights_types::WeightsType; use filesys::file_exists::file_exists; use filesys::path_to_string::path_to_string; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs; use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs; @@ -253,11 +256,11 @@ pub async fn process_job_sd( let stdout_output_file = work_temp_dir.path().join("sd_out.txt"); // run inference on loRA downloaded - sd_deps.inference_command.execute_inference(InferenceArgs { + let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs { work_dir: work_temp_dir.path().to_path_buf(), output_file: output_path.clone(), - stderr_output_file, - stdout_output_file, + stderr_output_file: &stderr_output_file, + stdout_output_file: &stdout_output_file, prompt:String::from("This is a green sign that says go this is a test prompt to test the model."), negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(), number_of_samples:20, @@ -272,14 +275,38 @@ pub async fn process_job_sd( batch_count: sd_args.maybe_batch_count.unwrap_or(1), }); + if !exit_status.is_success() { + error!("SD inference failed: {:?}", exit_status); + + let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status)); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + warn!("Captured stderr output: {}", contents); + + //match categorize_error(&contents) { + // Some(ProcessSingleJobError::FaceDetectionFailure) => { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + //} + } + return Err(error); + } + + // check if file exists if it does not error out.... let path = output_path.clone(); let file_path = format!("{}_{}.png", path_to_string(path), 0); - if file_exists(file_path.path()) == false { - return Err( - ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model")) - ); + if !file_exists(file_path.path()) { + error!("Output file did not exist: {:?}", file_path); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + error!("Captured stderr output: {}", contents); + } + + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model"))); } // If it worked and didn't fail! then we should save and create the weight. @@ -485,11 +512,11 @@ pub async fn process_job_lora( let stdout_output_file = work_temp_dir.path().join("sd_out.txt"); // run inference on loRA downloaded - sd_deps.inference_command.execute_inference(InferenceArgs { + let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs { work_dir: work_temp_dir.path().to_path_buf(), output_file: output_path.clone(), - stderr_output_file, - stdout_output_file, + stderr_output_file: &stderr_output_file, + stdout_output_file: &stdout_output_file, prompt:String::from("This is a green sign that says go this is a test prompt to test the model."), negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(), number_of_samples:20, @@ -504,6 +531,25 @@ pub async fn process_job_lora( batch_count: sd_args.maybe_batch_count.unwrap_or(1), }); + if !exit_status.is_success() { + error!("SD inference failed: {:?}", exit_status); + + let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status)); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + warn!("Captured stderr output: {}", contents); + + //match categorize_error(&contents) { + // Some(ProcessSingleJobError::FaceDetectionFailure) => { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + //} + } + return Err(error); + } + // check if file exists if it does not error out.... let path = output_path.clone(); let file_path = format!("{}_{}.png", path_to_string(path), 0); @@ -769,11 +815,11 @@ pub async fn process_job_inference( let inference_start_time = Instant::now(); - sd_deps.inference_command.execute_inference(InferenceArgs { + let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs { work_dir: work_temp_dir.path().to_path_buf(), output_file: output_path.clone(), - stderr_output_file, - stdout_output_file, + stderr_output_file: &stderr_output_file, + stdout_output_file: &stdout_output_file, prompt: prompt.clone(), negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(), number_of_samples, @@ -788,8 +834,27 @@ pub async fn process_job_inference( batch_count: sd_args.maybe_batch_count.unwrap_or(1), }); + if !exit_status.is_success() { + error!("SD inference failed: {:?}", exit_status); - // hack to check the directory before clean up. + let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status)); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + warn!("Captured stderr output: {}", contents); + + //match categorize_error(&contents) { + // Some(ProcessSingleJobError::FaceDetectionFailure) => { + // warn!("Face not detected in source image"); + // error = ProcessSingleJobError::FaceDetectionFailure; + // } + // _ => {} + //} + } + return Err(error); + } + + + // hack to check the directory before clean up. // let thirtyMinutes = 1800; // thread::sleep(Duration::from_secs(thirtyMinutes)); // upload media and create a record. diff --git a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/sd_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/sd_inference_command.rs index 5e8fc2ce63..9edd0035ea 100644 --- a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/sd_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/sd_inference_command.rs @@ -64,14 +64,14 @@ pub enum ExecutableOrCommand { Command(String), } -pub struct InferenceArgs { +pub struct InferenceArgs<'a> { /// --source_image: path to the input image (or video) /// --result_dir: path to directory work is performed pub work_dir: PathBuf, /// --result_file: path to final file output pub output_file: PathBuf, - pub stdout_output_file: PathBuf, - pub stderr_output_file: PathBuf, + pub stdout_output_file: &'a PathBuf, + pub stderr_output_file: &'a PathBuf, pub prompt: String, pub negative_prompt:String, pub number_of_samples:u32, @@ -272,8 +272,8 @@ impl StableDiffusionInferenceCommand { info!("stderr will be written to file: {}", path_to_string(args.stderr_output_file.clone())); - let stderr_file = File::create(&args.stderr_output_file)?; - let stdout_file = File::create(&args.stdout_output_file)?; + let stderr_file = File::create(args.stderr_output_file)?; + let stdout_file = File::create(args.stdout_output_file)?; config.stderr = Redirection::File(stderr_file); config.stdout = Redirection::File(stdout_file);