From 8017fef05a4f9e48dbe2a8be47da2dd3defeed95 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 20 Jul 2023 23:50:19 -0400 Subject: [PATCH] rename files across devices --- crates/lib/files/filesys/src/lib.rs | 1 + .../filesys/src/rename_across_devices.rs | 31 +++++++++++++++++++ .../vc/rvc_v2/pretrained_hubert_model.rs | 6 ++-- 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 crates/lib/files/filesys/src/rename_across_devices.rs diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index de851b96b1..77a085d899 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -24,3 +24,4 @@ pub mod file_exists; pub mod file_size; pub mod filename_concat; pub mod path_to_string; +pub mod rename_across_devices; diff --git a/crates/lib/files/filesys/src/rename_across_devices.rs b/crates/lib/files/filesys/src/rename_across_devices.rs new file mode 100644 index 0000000000..f990bc671f --- /dev/null +++ b/crates/lib/files/filesys/src/rename_across_devices.rs @@ -0,0 +1,31 @@ +use std::path::Path; + +/// Rename a file. +/// Ordinary rename will fail in Linux if it is across physical devices. +/// This function will perform a copy followed by delete in that case. +pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> std::io::Result<()> { + let result = std::fs::rename(&from, &to); + + let err = match result.as_ref() { + Ok(_) => return Ok(()), // Rename succeeded. + Err(err) => err, + }; + + match err.raw_os_error() { + Some(18) => { + // NO-OP: pass-through + // NB: We can clean this up in the future when `err.kind() == ErrorKind::CrossDevices` + // stabilizes. For now we can assume code "18" means we're trying to rename across devices. + } + _ => { + // Something else happened. Return original error. + return result; + } + } + + let _num_bytes = std::fs::copy(&from, &to)?; + + std::fs::remove_file(&from)?; + + Ok(()) +} diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs index d9204dba4a..031e19a053 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs @@ -1,13 +1,13 @@ use anyhow::anyhow; use cloud_storage::bucket_client::BucketClient; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; use errors::AnyhowResult; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use log::{error, info}; use std::path::PathBuf; -use tempdir::TempDir; -use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; +use filesys::rename_across_devices::rename_across_devices; #[derive(Clone)] pub struct PretrainedHubertModel { @@ -71,7 +71,7 @@ impl PretrainedHubertModel { info!("Renaming hubert file from {:?} to {:?}!", &temp_path, &self.filesystem_path); - std::fs::rename(&temp_path, &self.filesystem_path) + rename_across_devices(&temp_path, &self.filesystem_path) .map_err(|e| { error!("could rename hubert model on disk: {:?}", e); safe_delete_temp_directory(&temp_dir);