From becc3e2c8f6ab39d6bca7ae283c2f11d843b5ba1 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 21 Aug 2024 16:33:08 -0400 Subject: [PATCH] shared function to delete all files --- .../files/filesys/src/file_deletion/mod.rs | 3 +- .../safe_delete_all_files_and_directories.rs | 39 ++++++++++++++ .../src/file_deletion/safe_delete_file.rs | 3 +- .../process_video_style_transfer_job.rs | 53 +++++++++---------- 4 files changed, 68 insertions(+), 30 deletions(-) create mode 100644 crates/lib/files/filesys/src/file_deletion/safe_delete_all_files_and_directories.rs diff --git a/crates/lib/files/filesys/src/file_deletion/mod.rs b/crates/lib/files/filesys/src/file_deletion/mod.rs index 9e23350407..64a8718cfd 100644 --- a/crates/lib/files/filesys/src/file_deletion/mod.rs +++ b/crates/lib/files/filesys/src/file_deletion/mod.rs @@ -1,4 +1,5 @@ -pub mod safe_delete_possible_file; pub mod safe_delete_directory; pub mod safe_delete_file; +pub mod safe_delete_all_files_and_directories; +pub mod safe_delete_possible_file; pub mod safe_recursively_delete_files; \ No newline at end of file diff --git a/crates/lib/files/filesys/src/file_deletion/safe_delete_all_files_and_directories.rs b/crates/lib/files/filesys/src/file_deletion/safe_delete_all_files_and_directories.rs new file mode 100644 index 0000000000..66845186d5 --- /dev/null +++ b/crates/lib/files/filesys/src/file_deletion/safe_delete_all_files_and_directories.rs @@ -0,0 +1,39 @@ +use std::path::Path; + +use log::warn; + +use crate::file_deletion::safe_delete_directory::safe_delete_directory; +use crate::file_deletion::safe_delete_file::safe_delete_file; +use crate::file_deletion::safe_recursively_delete_files::safe_recursively_delete_files; + +/// Safely deletes all files and directories passed to the function without panicking. Errors are logged. +/// Directories are recursively deleted before being removed. +/// This is an infallible, idempotent function. +/// +/// Note on usage: since this is a generic function with only one generic type, `P`, you can only pass +/// a list of paths that are all the same type. If you have a mix of `PathBuf` and `&Path`, it's best to +/// convert the PathBuf to &Path references. Mixing other `AsRef` types will be more complicated, +/// so it may be worth calling `as_ref()` on them, e.g.: +/// +/// ```rust +/// use std::path::{Path, PathBuf}; +/// use filesys::file_deletion::safe_delete_all_files_and_directories::safe_delete_all_files_and_directories; +/// +/// safe_delete_all_files_and_directories(&[ +/// Path::new("1234_foo_bar_baz.bash"), +/// &PathBuf::from("1234_foo_bar_baz.bin"), +/// ]); +/// ``` +pub fn safe_delete_all_files_and_directories>(paths: &[P]) { + for path in paths { + let p = path.as_ref(); + if p.is_file() { + safe_delete_file(p); + } else if p.is_dir() { + safe_recursively_delete_files(p); + safe_delete_directory(p); + } else { + warn!("Path {:?} is neither a file nor a directory", p); + } + } +} diff --git a/crates/lib/files/filesys/src/file_deletion/safe_delete_file.rs b/crates/lib/files/filesys/src/file_deletion/safe_delete_file.rs index 1718739f1f..bc0c92d090 100644 --- a/crates/lib/files/filesys/src/file_deletion/safe_delete_file.rs +++ b/crates/lib/files/filesys/src/file_deletion/safe_delete_file.rs @@ -1,4 +1,3 @@ -use std::fs; use std::path::Path; use log::info; @@ -8,7 +7,7 @@ use log::warn; /// This is an infallible, idempotent function. pub fn safe_delete_file>(file_path: P) { let printable_name = file_path.as_ref().to_str().unwrap_or("bad filename"); - match fs::remove_file(&file_path) { + match std::fs::remove_file(&file_path) { Ok(_) => info!("Temp file deleted: {}", printable_name), Err(e) => warn!("Could not delete temp file {:?} (not a fatal error): {:?}", printable_name, e), diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/process_video_style_transfer_job.rs b/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/process_video_style_transfer_job.rs index 399f836179..6a93fcc136 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/process_video_style_transfer_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/video_style_transfer/process_video_style_transfer_job.rs @@ -23,6 +23,7 @@ use enums::by_table::prompts::prompt_type::PromptType; use enums::common::job_status_plus::JobStatusPlus; use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; +use filesys::file_deletion::safe_delete_all_files_and_directories::safe_delete_all_files_and_directories; use filesys::file_exists::file_exists; use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; @@ -544,11 +545,14 @@ pub async fn process_video_style_transfer_job(deps: &JobDependencies, job: &Avai Err(err) => { error!("Error validating and saving results: {:?}", err); safe_delete_all_input_videos(&videos); - safe_delete_files_and_directories(&[ + + // TODO(bt,2024-03-01): Do we really want to delete the workflow, models, etc.? + safe_delete_all_files_and_directories(&[ positive_prompt_file.as_path(), &negative_prompt_file, &travel_prompt_file, &preview_frames_dir, + workflow_path.as_ref(), &output_dir, work_temp_dir.path(), ]); @@ -571,10 +575,6 @@ pub async fn process_video_style_transfer_job(deps: &JobDependencies, job: &Avai safe_delete_file(&stdout_output_file); safe_delete_all_input_videos(&videos); - // TODO(bt,2024-03-01): Do we really want to delete the workflow, models, etc.? - - safe_delete_file(&workflow_path); - // TODO(bt,2024-04-21): Not sure we want to delete the LoRA? if let Some(lora_path) = maybe_lora_path { safe_delete_file(lora_path); @@ -584,11 +584,16 @@ pub async fn process_video_style_transfer_job(deps: &JobDependencies, job: &Avai safe_delete_file(ipa_path.ipa_image_path); } - let output_dir = root_comfy_path.join("output"); - safe_recursively_delete_files(&output_dir); - - // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - safe_delete_directory(&work_temp_dir); + // TODO(bt,2024-03-01): Do we really want to delete the workflow, models, etc.? + safe_delete_all_files_and_directories(&[ + positive_prompt_file.as_path(), + &negative_prompt_file, + &travel_prompt_file, + &preview_frames_dir, + workflow_path.as_ref(), + &output_dir, + work_temp_dir.path(), + ]); // ==================== DONE ==================== // @@ -610,20 +615,6 @@ pub async fn process_video_style_transfer_job(deps: &JobDependencies, job: &Avai }) } -fn safe_delete_files_and_directories>(paths: &[P]) { - for path in paths { - let p = path.as_ref(); - if p.is_file() { - safe_delete_file(p); - } else if p.is_dir() { - safe_recursively_delete_files(p); - safe_delete_directory(p); - } else { - warn!("Path {:?} is neither a file nor a directory", p); - } - } -} - fn safe_delete_all_input_videos(videos: &VideoPathing) { safe_delete_primary_videos(&videos.primary_video); @@ -641,13 +632,21 @@ fn safe_delete_all_input_videos(videos: &VideoPathing) { } fn safe_delete_primary_videos(video: &PrimaryInputVideoAndPaths) { - safe_delete_file(&video.original_download_path); + safe_delete_file(&video.comfy_input_video_path); safe_delete_file(&video.comfy_output_video_path); + safe_delete_file(&video.original_download_path); + safe_delete_file(&video.trimmed_wav_audio_path); safe_delete_file(video.video_to_watermark()); safe_delete_file(video.get_final_video_to_upload()); safe_delete_file(video.get_non_watermarked_video_to_upload()); - if let Some(processed_path) = &video.maybe_trimmed_resampled_path { - safe_delete_file(processed_path); + if let Some(path) = &video.maybe_trimmed_resampled_path { + safe_delete_file(path); + } + if let Some(path) = &video.audio_restored_video_path { + safe_delete_file(path); + } + if let Some(path) = &video.watermarked_video_path{ + safe_delete_file(path); } }