rename files across fs devices

crates/service/job/inference_job/src/job_dependencies.rs

@@ -144,10 +144,6 @@ pub struct FileSystemDetails {
   /// redeploying.
   pub maybe_pause_file: Option<PathBuf>,
 
-  /// Directory to store long-term downloads (models)
-  /// (In prod, typically model files from GCS / NFS PVC mount)
-  pub directory_long_lived_downloads: PathBuf,
-
   /// Temporary directory for storing downloads
   /// Creates temp directories for scratch files
   pub scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator,
@@ -161,6 +157,8 @@ pub struct FileSystemDetails {
   pub scoped_temp_dir_creator_for_work: ScopedTempDirCreator,
 
   /// Organizes downloaded files
+  /// Directory to store long-term downloads (models)
+  /// (In prod, typically model files from GCS / NFS PVC mount)
   pub semi_persistent_cache: SemiPersistentCacheDir,
 }
 

crates/service/job/inference_job/src/main.rs

@@ -218,8 +218,6 @@ async fn main() -> AnyhowResult<()> {
     },
     fs: FileSystemDetails {
       maybe_pause_file: easyenv::get_env_pathbuf_optional("PAUSE_FILE"),
-      directory_long_lived_downloads: easyenv::get_env_pathbuf_or_default(
-        "DIR_LONG_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_long_lived")),
       scoped_temp_dir_creator_for_short_lived_downloads: ScopedTempDirCreator::for_directory(
         easyenv::get_env_pathbuf_or_default(
           "SCOPED_TEMP_DIR_SHORT_LIVED_DOWNLOADS", PathBuf::from("/tmp/downloads_short_lived"))),
@@ -314,13 +312,11 @@ fn set_up_directories(job_dependencies: &JobDependencies) -> AnyhowResult<()> {
   if !job_dependencies.job.info.container.server_environment.is_deployed_in_production() {
     warn!("Creating directories for non-production / development only!");
 
-    create_dir_all_if_missing(&fs.directory_long_lived_downloads)?;
     create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?;
     create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?;
     create_dir_all_if_missing(&fs.scoped_temp_dir_creator_for_work.get_base_dir())?;
   }
 
-  check_directory_exists(&fs.directory_long_lived_downloads)?;
   check_directory_exists(fs.scoped_temp_dir_creator_for_long_lived_downloads.get_base_dir())?;
   check_directory_exists(fs.scoped_temp_dir_creator_for_short_lived_downloads.get_base_dir())?;
   check_directory_exists(fs.scoped_temp_dir_creator_for_work.get_base_dir())?;

crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs

@@ -1,9 +1,10 @@
 use std::path::Path;
 
-use log::{info, warn};
+use log::{error, info, warn};
 
 use cloud_storage::bucket_client::BucketClient;
 use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::rename_across_devices::rename_across_devices;
 use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
 
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -56,10 +57,11 @@ pub async fn maybe_download_file_from_bucket(
   info!("Renaming {} temp file from {:?} to {:?}!",
     name_or_description_of_file, &temp_path, &final_filesystem_file_path);
 
-  std::fs::rename(&temp_path, &final_filesystem_file_path)
-      .map_err(|e| {
+  rename_across_devices(&temp_path, &final_filesystem_file_path)
+      .map_err(|err| {
+        error!("could not rename on disk: {:?}", err);
         safe_delete_temp_directory(&temp_dir);
-        ProcessSingleJobError::from_io_error(e)
+        ProcessSingleJobError::from_io_error(err)
       })?;
 
   info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path);