From f2e75f863712464cb735fb4be660eec0c1dc6bdb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Nov 2023 18:46:00 -0500 Subject: [PATCH] rename files across fs devices --- .../service/job/inference_job/src/job_dependencies.rs | 6 ++---- crates/service/job/inference_job/src/main.rs | 4 ---- .../src/util/maybe_download_file_from_bucket.rs | 10 ++++++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 7b99bad23a..448554e1ee 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -144,10 +144,6 @@ pub struct FileSystemDetails { /// redeploying. pub maybe_pause_file: Option, - /// Directory to store long-term downloads (models) - /// (In prod, typically model files from GCS / NFS PVC mount) - pub directory_long_lived_downloads: PathBuf, - /// Temporary directory for storing downloads /// Creates temp directories for scratch files pub scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator, @@ -161,6 +157,8 @@ pub struct FileSystemDetails { pub scoped_temp_dir_creator_for_work: ScopedTempDirCreator, /// Organizes downloaded files + /// Directory to store long-term downloads (models) + /// (In prod, typically model files from GCS / NFS PVC mount) pub semi_persistent_cache: SemiPersistentCacheDir, } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 43a7ab5569..ceff31645a 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -218,8 +218,6 @@ async fn main() -> AnyhowResult<()> { }, fs: FileSystemDetails { maybe_pause_file: easyenv::get_env_pathbuf_optional("PAUSE_FILE"), - directory_long_lived_downloads: easyenv::get_env_pathbuf_or_default( - "DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived")), scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory( easyenv::get_env_pathbuf_or_default( "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_short_lived"))), @@ -314,13 +312,11 @@ fn set_up_directories(job_dependencies: &JobDependencies) -> AnyhowResult<()> { if !job_dependencies.job.info.container.server_environment.is_deployed_in_production() { warn!("Creating directories for non-production / development only!"); - create_dir_all_if_missing(&fs.directory_long_lived_downloads)?; create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?; create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?; create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_work.get_base_dir())?; } - check_directory_exists(&fs.directory_long_lived_downloads)?; check_directory_exists(fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?; check_directory_exists(fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?; check_directory_exists(fs.scoped_temp_dir_creator_for_work.get_base_dir())?; diff --git a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs index 98a49512ca..60626e7985 100644 --- a/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs +++ b/crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs @@ -1,9 +1,10 @@ use std::path::Path; -use log::{info, warn}; +use log::{error, info, warn}; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::rename_across_devices::rename_across_devices; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -56,10 +57,11 @@ pub async fn maybe_download_file_from_bucket( info!("Renaming {} temp file from {:?} to {:?}!", name_or_description_of_file, &temp_path, &final_filesystem_file_path); - std::fs::rename(&temp_path, &final_filesystem_file_path) - .map_err(|e| { + rename_across_devices(&temp_path, &final_filesystem_file_path) + .map_err(|err| { + error!("could not rename on disk: {:?}", err); safe_delete_temp_directory(&temp_dir); - ProcessSingleJobError::from_io_error(e) + ProcessSingleJobError::from_io_error(err) })?; info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path);