daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-05 17:58:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

output stderr on failure

Browse Source
This commit is contained in:
Brandon Thomas committed 2024-02-02 23:28:43 -05:00
1 parent c8fa722b3d
commit 4e20f63f87
2 files changed
+85 -20
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job.rs
+80 -15
View File
@@ -1,9 +1,10 @@
use std::fs::read_to_string;
use std::path::PathBuf;
use std::time::{Duration, Instant};
use actix_web::dev::ResourcePath;
use anyhow::anyhow;
use log::{error, info};
use log::{error, info, warn};
use serde_json;
use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
@@ -21,6 +22,8 @@ use enums::by_table::model_weights::weights_category::WeightsCategory;
use enums::by_table::model_weights::weights_types::WeightsType;
use filesys::file_exists::file_exists;
use filesys::path_to_string::path_to_string;
use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
use filesys::safe_delete_temp_file::safe_delete_temp_file;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
@@ -253,11 +256,11 @@ pub async fn process_job_sd(
let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
// run inference on loRA downloaded
sd_deps.inference_command.execute_inference(InferenceArgs {
let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
work_dir: work_temp_dir.path().to_path_buf(),
output_file: output_path.clone(),
stderr_output_file,
stdout_output_file,
stderr_output_file: &stderr_output_file,
stdout_output_file: &stdout_output_file,
prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
number_of_samples:20,
@@ -272,14 +275,38 @@ pub async fn process_job_sd(
batch_count: sd_args.maybe_batch_count.unwrap_or(1),
});
if !exit_status.is_success() {
error!("SD inference failed: {:?}", exit_status);
let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
if let Ok(contents) = read_to_string(&stderr_output_file) {
warn!("Captured stderr output: {}", contents);
//match categorize_error(&contents) {
// Some(ProcessSingleJobError::FaceDetectionFailure) => {
// warn!("Face not detected in source image");
// error = ProcessSingleJobError::FaceDetectionFailure;
// }
// _ => {}
//}
}
return Err(error);
}
// check if file exists if it does not error out....
let path = output_path.clone();
let file_path = format!("{}_{}.png", path_to_string(path), 0);
if file_exists(file_path.path()) == false {
return Err(
ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model"))
);
if !file_exists(file_path.path()) {
error!("Output file did not exist: {:?}", file_path);
if let Ok(contents) = read_to_string(&stderr_output_file) {
error!("Captured stderr output: {}", contents);
}
return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Upload Not a SD Model")));
}
// If it worked and didn't fail! then we should save and create the weight.
@@ -485,11 +512,11 @@ pub async fn process_job_lora(
let stdout_output_file = work_temp_dir.path().join("sd_out.txt");
// run inference on loRA downloaded
sd_deps.inference_command.execute_inference(InferenceArgs {
let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
work_dir: work_temp_dir.path().to_path_buf(),
output_file: output_path.clone(),
stderr_output_file,
stdout_output_file,
stderr_output_file: &stderr_output_file,
stdout_output_file: &stdout_output_file,
prompt:String::from("This is a green sign that says go this is a test prompt to test the model."),
negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
number_of_samples:20,
@@ -504,6 +531,25 @@ pub async fn process_job_lora(
batch_count: sd_args.maybe_batch_count.unwrap_or(1),
});
if !exit_status.is_success() {
error!("SD inference failed: {:?}", exit_status);
let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
if let Ok(contents) = read_to_string(&stderr_output_file) {
warn!("Captured stderr output: {}", contents);
//match categorize_error(&contents) {
// Some(ProcessSingleJobError::FaceDetectionFailure) => {
// warn!("Face not detected in source image");
// error = ProcessSingleJobError::FaceDetectionFailure;
// }
// _ => {}
//}
}
return Err(error);
}
// check if file exists if it does not error out....
let path = output_path.clone();
let file_path = format!("{}_{}.png", path_to_string(path), 0);
@@ -769,11 +815,11 @@ pub async fn process_job_inference(
let inference_start_time = Instant::now();
sd_deps.inference_command.execute_inference(InferenceArgs {
let exit_status = sd_deps.inference_command.execute_inference(InferenceArgs {
work_dir: work_temp_dir.path().to_path_buf(),
output_file: output_path.clone(),
stderr_output_file,
stdout_output_file,
stderr_output_file: &stderr_output_file,
stdout_output_file: &stdout_output_file,
prompt: prompt.clone(),
negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(),
number_of_samples,
@@ -788,8 +834,27 @@ pub async fn process_job_inference(
batch_count: sd_args.maybe_batch_count.unwrap_or(1),
});
if !exit_status.is_success() {
error!("SD inference failed: {:?}", exit_status);
// hack to check the directory before clean up.
let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", exit_status));
if let Ok(contents) = read_to_string(&stderr_output_file) {
warn!("Captured stderr output: {}", contents);
//match categorize_error(&contents) {
// Some(ProcessSingleJobError::FaceDetectionFailure) => {
// warn!("Face not detected in source image");
// error = ProcessSingleJobError::FaceDetectionFailure;
// }
// _ => {}
//}
}
return Err(error);
}
// hack to check the directory before clean up.
// let thirtyMinutes = 1800;
// thread::sleep(Duration::from_secs(thirtyMinutes));
// upload media and create a record.
crates/service/job/inference_job/src/job/job_types/image_generation/sd/sd_inference_command.rs
+5 -5
View File
@@ -64,14 +64,14 @@ pub enum ExecutableOrCommand {
Command(String),
}
pub struct InferenceArgs {
pub struct InferenceArgs<'a> {
/// --source_image: path to the input image (or video)
/// --result_dir: path to directory work is performed
pub work_dir: PathBuf,
/// --result_file: path to final file output
pub output_file: PathBuf,
pub stdout_output_file: PathBuf,
pub stderr_output_file: PathBuf,
pub stdout_output_file: &'a PathBuf,
pub stderr_output_file: &'a PathBuf,
pub prompt: String,
pub negative_prompt:String,
pub number_of_samples:u32,
@@ -272,8 +272,8 @@ impl StableDiffusionInferenceCommand {
info!("stderr will be written to file: {}", path_to_string(args.stderr_output_file.clone()));
let stderr_file = File::create(&args.stderr_output_file)?;
let stdout_file = File::create(&args.stdout_output_file)?;
let stderr_file = File::create(args.stderr_output_file)?;
let stdout_file = File::create(args.stdout_output_file)?;
config.stderr = Redirection::File(stderr_file);
config.stdout = Redirection::File(stdout_file);