From 5a79f20deff33ee37a74fd2ca613f60f3c1032c4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Jun 2022 03:19:59 -0400 Subject: [PATCH] Move filesystem utils into container_common:: --- .../src/filesystem/check_directory_exists.rs | 17 +++++++ .../src/filesystem/check_file_exists.rs | 17 +++++++ .../container_common/src/filesystem/mod.rs | 4 ++ .../filesystem/safe_delete_temp_directory.rs | 17 +++++++ .../src/filesystem/safe_delete_temp_file.rs | 17 +++++++ crates/lib/container_common/src/lib.rs | 1 + .../storyteller_web/src/tts_download_job.rs | 7 +-- .../storyteller_web/src/tts_inference_job.rs | 8 ++-- .../storyteller_web/src/util/filesystem.rs | 47 ------------------- .../service/storyteller_web/src/util/mod.rs | 1 - .../storyteller_web/src/w2l_download_job.rs | 8 ++-- .../storyteller_web/src/w2l_inference_job.rs | 8 ++-- 12 files changed, 89 insertions(+), 63 deletions(-) create mode 100644 crates/lib/container_common/src/filesystem/check_directory_exists.rs create mode 100644 crates/lib/container_common/src/filesystem/check_file_exists.rs create mode 100644 crates/lib/container_common/src/filesystem/mod.rs create mode 100644 crates/lib/container_common/src/filesystem/safe_delete_temp_directory.rs create mode 100644 crates/lib/container_common/src/filesystem/safe_delete_temp_file.rs delete mode 100644 crates/service/storyteller_web/src/util/filesystem.rs diff --git a/crates/lib/container_common/src/filesystem/check_directory_exists.rs b/crates/lib/container_common/src/filesystem/check_directory_exists.rs new file mode 100644 index 0000000000..7b86b92850 --- /dev/null +++ b/crates/lib/container_common/src/filesystem/check_directory_exists.rs @@ -0,0 +1,17 @@ +use anyhow::bail; +use crate::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { + if !path.exists() { + bail!("Path doesn't exist: {:?}", path); + } + if !path.is_dir() { + bail!("Path isn't a directory: {:?}", path); + } + Ok(()) +} diff --git a/crates/lib/container_common/src/filesystem/check_file_exists.rs b/crates/lib/container_common/src/filesystem/check_file_exists.rs new file mode 100644 index 0000000000..1932422a27 --- /dev/null +++ b/crates/lib/container_common/src/filesystem/check_file_exists.rs @@ -0,0 +1,17 @@ +use anyhow::bail; +use crate::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +pub fn check_file_exists(path: &PathBuf) -> AnyhowResult<()> { + if !path.exists() { + bail!("Path doesn't exist: {:?}", path); + } + if !path.is_file() { + bail!("Path isn't a file: {:?}", path); + } + Ok(()) +} diff --git a/crates/lib/container_common/src/filesystem/mod.rs b/crates/lib/container_common/src/filesystem/mod.rs new file mode 100644 index 0000000000..edd4fc339f --- /dev/null +++ b/crates/lib/container_common/src/filesystem/mod.rs @@ -0,0 +1,4 @@ +pub mod check_directory_exists; +pub mod check_file_exists; +pub mod safe_delete_temp_directory; +pub mod safe_delete_temp_file; \ No newline at end of file diff --git a/crates/lib/container_common/src/filesystem/safe_delete_temp_directory.rs b/crates/lib/container_common/src/filesystem/safe_delete_temp_directory.rs new file mode 100644 index 0000000000..a796358347 --- /dev/null +++ b/crates/lib/container_common/src/filesystem/safe_delete_temp_directory.rs @@ -0,0 +1,17 @@ +use anyhow::bail; +use crate::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +pub fn safe_delete_temp_directory>(directory_path: P) { + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + let printable_name = directory_path.as_ref().to_str().unwrap_or("bad directory"); + match fs::remove_dir_all(&directory_path) { + Ok(_) => info!("Temp directory deleted: {}", printable_name), + Err(e) => warn!("Could not delete temp directory{:?} (not a fatal error): {:?}", + printable_name, e), + } +} diff --git a/crates/lib/container_common/src/filesystem/safe_delete_temp_file.rs b/crates/lib/container_common/src/filesystem/safe_delete_temp_file.rs new file mode 100644 index 0000000000..14d57c1f78 --- /dev/null +++ b/crates/lib/container_common/src/filesystem/safe_delete_temp_file.rs @@ -0,0 +1,17 @@ +use anyhow::bail; +use crate::anyhow_result::AnyhowResult; +use log::info; +use log::warn; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +pub fn safe_delete_temp_file>(file_path: P) { + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + let printable_name = file_path.as_ref().to_str().unwrap_or("bad filename"); + match fs::remove_file(&file_path) { + Ok(_) => info!("Temp file deleted: {}", printable_name), + Err(e) => warn!("Could not delete temp file {:?} (not a fatal error): {:?}", + printable_name, e), + } +} diff --git a/crates/lib/container_common/src/lib.rs b/crates/lib/container_common/src/lib.rs index 49c1946532..7db94caa9d 100644 --- a/crates/lib/container_common/src/lib.rs +++ b/crates/lib/container_common/src/lib.rs @@ -1,6 +1,7 @@ pub mod anyhow_result; pub mod collections; pub mod files; +pub mod filesystem; pub mod i18n; pub mod numerics; pub mod thread; diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs index d9ef14342a..19a2b4fa33 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/storyteller_web/src/tts_download_job.rs @@ -20,6 +20,10 @@ use chrono::Utc; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; +use container_common::filesystem::check_directory_exists::check_directory_exists; +use container_common::filesystem::check_file_exists::check_file_exists; +use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use crate::common_env::CommonEnv; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; @@ -33,9 +37,6 @@ use crate::script_execution::talknet_model_check_command::TalknetModelCheckComma use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; -use crate::util::filesystem::safe_delete_temp_directory; -use crate::util::filesystem::safe_delete_temp_file; -use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index 76aa83b44b..cc66b181ed 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -22,6 +22,10 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::collections::multiple_random_from_vec::multiple_random_from_vec; +use container_common::filesystem::check_directory_exists::check_directory_exists; +use container_common::filesystem::check_file_exists::check_file_exists; +use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use container_common::token::random_uuid::generate_random_uuid; use crate::common_env::CommonEnv; use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; @@ -29,10 +33,6 @@ use crate::script_execution::tacotron_inference_command::TacotronInferenceComman use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; -use crate::util::filesystem::check_directory_exists; -use crate::util::filesystem::check_file_exists; -use crate::util::filesystem::safe_delete_temp_directory; -use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::hashing::hash_string_sha2::hash_string_sha2; use crate::util::jobs::cache_miss_strategizer::CacheMissStrategizer; diff --git a/crates/service/storyteller_web/src/util/filesystem.rs b/crates/service/storyteller_web/src/util/filesystem.rs deleted file mode 100644 index b4553ed95a..0000000000 --- a/crates/service/storyteller_web/src/util/filesystem.rs +++ /dev/null @@ -1,47 +0,0 @@ -use anyhow::bail; -use container_common::anyhow_result::AnyhowResult; -use log::info; -use log::warn; -use std::fs; -use std::path::Path; -use std::path::PathBuf; - -pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { - if !path.exists() { - bail!("Path doesn't exist: {:?}", path); - } - if !path.is_dir() { - bail!("Path isn't a directory: {:?}", path); - } - Ok(()) -} - -pub fn check_file_exists(path: &PathBuf) -> AnyhowResult<()> { - if !path.exists() { - bail!("Path doesn't exist: {:?}", path); - } - if !path.is_file() { - bail!("Path isn't a file: {:?}", path); - } - Ok(()) -} - -pub fn safe_delete_temp_file>(file_path: P) { - // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - let printable_name = file_path.as_ref().to_str().unwrap_or("bad filename"); - match fs::remove_file(&file_path) { - Ok(_) => info!("Temp file deleted: {}", printable_name), - Err(e) => warn!("Could not delete temp file {:?} (not a fatal error): {:?}", - printable_name, e), - } -} - -pub fn safe_delete_temp_directory>(directory_path: P) { - // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... - let printable_name = directory_path.as_ref().to_str().unwrap_or("bad directory"); - match fs::remove_dir_all(&directory_path) { - Ok(_) => info!("Temp directory deleted: {}", printable_name), - Err(e) => warn!("Could not delete temp directory{:?} (not a fatal error): {:?}", - printable_name, e), - } -} diff --git a/crates/service/storyteller_web/src/util/mod.rs b/crates/service/storyteller_web/src/util/mod.rs index da7e8cad13..569e2af8ec 100644 --- a/crates/service/storyteller_web/src/util/mod.rs +++ b/crates/service/storyteller_web/src/util/mod.rs @@ -3,7 +3,6 @@ pub mod caching; pub mod delete_role_disambiguation; pub mod email_to_gravatar; pub mod encrypted_sort_id; -pub mod filesystem; pub mod hashing; pub mod jobs; pub mod markdown_to_html; diff --git a/crates/service/storyteller_web/src/w2l_download_job.rs b/crates/service/storyteller_web/src/w2l_download_job.rs index 370bbf8783..2e0aabc75e 100644 --- a/crates/service/storyteller_web/src/w2l_download_job.rs +++ b/crates/service/storyteller_web/src/w2l_download_job.rs @@ -20,6 +20,10 @@ use chrono::Utc; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; +use container_common::filesystem::check_directory_exists::check_directory_exists; +use container_common::filesystem::check_file_exists::check_file_exists; +use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use crate::common_env::CommonEnv; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; @@ -35,10 +39,6 @@ use crate::script_execution::imagemagick_generate_preview_image_command::Imagema use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError}; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_paths::hash_to_bucket_path; -use crate::util::filesystem::check_directory_exists; -use crate::util::filesystem::check_file_exists; -use crate::util::filesystem::safe_delete_temp_directory; -use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; diff --git a/crates/service/storyteller_web/src/w2l_inference_job.rs b/crates/service/storyteller_web/src/w2l_inference_job.rs index 6c2ce2178d..21e082395a 100644 --- a/crates/service/storyteller_web/src/w2l_inference_job.rs +++ b/crates/service/storyteller_web/src/w2l_inference_job.rs @@ -20,6 +20,10 @@ use chrono::Utc; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; +use container_common::filesystem::check_directory_exists::check_directory_exists; +use container_common::filesystem::check_file_exists::check_file_exists; +use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use crate::common_env::CommonEnv; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; @@ -36,10 +40,6 @@ use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; -use crate::util::filesystem::check_directory_exists; -use crate::util::filesystem::check_file_exists; -use crate::util::filesystem::safe_delete_temp_directory; -use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger;