diff --git a/crates/lib/files/filesys/src/rename_across_devices.rs b/crates/lib/files/filesys/src/rename_across_devices.rs index f990bc671f..91a5ae9e67 100644 --- a/crates/lib/files/filesys/src/rename_across_devices.rs +++ b/crates/lib/files/filesys/src/rename_across_devices.rs @@ -3,6 +3,7 @@ use std::path::Path; /// Rename a file. /// Ordinary rename will fail in Linux if it is across physical devices. /// This function will perform a copy followed by delete in that case. +/// In both cases, this function will overwrite the destination if it already exists. pub fn rename_across_devices, Q: AsRef>(from: P, to: Q) -> std::io::Result<()> { let result = std::fs::rename(&from, &to); diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index e810dc46d4..19c0d958c9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -16,7 +16,7 @@ use mysql_queries::queries::voice_conversion::results::get_voice_conversion_resu use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; pub struct AudioFile { @@ -94,17 +94,17 @@ pub async fn download_audio_file( let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin"); - maybe_download_file_from_bucket( - "audio file", - &downloaded_filesystem_path, - &bucket_object_path, - public_bucket_client, + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "audio file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, job_progress_reporter, - "downloading", - job.id.0, - &temp_dir_creator, - None, - ).await?; + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; Ok(AudioFile { filesystem_path: downloaded_filesystem_path, diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 3ad2e15ab9..6ac3b4d3e9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -14,7 +14,7 @@ use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_m use tokens::tokens::media_uploads::MediaUploadToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; pub struct ImageFile { @@ -65,17 +65,17 @@ pub async fn download_image_file( let downloaded_filesystem_path = work_temp_dir.path().join("image.bin"); - maybe_download_file_from_bucket( - "image file", - &downloaded_filesystem_path, - &bucket_object_path, - public_bucket_client, + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "image file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, job_progress_reporter, - "downloading", - job.id.0, - &temp_dir_creator, - None, - ).await?; + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; Ok(ImageFile { filesystem_path: downloaded_filesystem_path, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs index 60a43fe7a6..536c25d260 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs @@ -6,7 +6,7 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; pub struct StaticDependencies { pub waveglow_vocoder_model_fs_path : PathBuf, @@ -30,17 +30,17 @@ pub async fn download_static_dependencies<'a>( let waveglow_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&waveglow_vocoder_model_filename); let waveglow_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename); - maybe_download_file_from_bucket( - "waveglow vocoder model", - &waveglow_vocoder_model_fs_path, - &waveglow_vocoder_model_object_path, - &job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading vocoder (1 of 3)", - job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, - Some(1000), - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "waveglow vocoder model", + final_filesystem_file_path: &waveglow_vocoder_model_fs_path, + bucket_object_path: &waveglow_vocoder_model_object_path, + bucket_client: &job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading vocoder (1 of 3)", + job_id: job.id.0, + scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + maybe_existing_file_minimum_size_required: Some(1000), + }).await?; waveglow_vocoder_model_fs_path }; @@ -53,17 +53,17 @@ pub async fn download_static_dependencies<'a>( let hifigan_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_vocoder_model_filename); let hifigan_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename); - maybe_download_file_from_bucket( - "hifigan vocoder model", - &hifigan_vocoder_model_fs_path, - &hifigan_vocoder_model_object_path, - &job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading vocoder (2 of 3)", - job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, - Some(1000), - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "hifigan vocoder model", + final_filesystem_file_path: &hifigan_vocoder_model_fs_path, + bucket_object_path: &hifigan_vocoder_model_object_path, + bucket_client: &job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading vocoder (2 of 3)", + job_id: job.id.0, + scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + maybe_existing_file_minimum_size_required: Some(1000), + }).await?; hifigan_vocoder_model_fs_path }; @@ -76,17 +76,17 @@ pub async fn download_static_dependencies<'a>( let hifigan_superres_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_superres_vocoder_model_filename); let hifigan_superres_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename); - maybe_download_file_from_bucket( - "hifigan superres vocoder model", - &hifigan_superres_vocoder_model_fs_path, - &hifigan_superres_vocoder_model_object_path, - &job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading vocoder (3 of 3)", - job.id.0, - &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, - Some(1000), - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "hifigan superres vocoder model", + final_filesystem_file_path: &hifigan_superres_vocoder_model_fs_path, + bucket_object_path: &hifigan_superres_vocoder_model_object_path, + bucket_client: &job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading vocoder (3 of 3)", + job_id: job.id.0, + scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + maybe_existing_file_minimum_size_required: Some(1000), + }).await?; hifigan_superres_vocoder_model_fs_path }; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index f5fbbf6457..17e1e4a594 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -29,7 +29,7 @@ use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_s use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor}; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; /// Text starting with this will be treated as a test request. /// This allows the request to bypass the model cache and query the latest TTS model. @@ -119,17 +119,17 @@ async fn process_job_with_cleanup( let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token); let custom_vocoder_object_path = args.job_dependencies.buckets.bucket_path_unifier.vocoder_path(&vocoder.vocoder_private_bucket_hash); - maybe_download_file_from_bucket( - "custom vocoder", - &custom_vocoder_fs_path, - &custom_vocoder_object_path, - &args.job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading user vocoder", - job.id.0, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, - None, - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "custom vocoder", + final_filesystem_file_path: &custom_vocoder_fs_path, + bucket_object_path: &custom_vocoder_object_path, + bucket_client: &args.job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading user vocoder", + job_id: job.id.0, + scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + maybe_existing_file_minimum_size_required: None, + }).await?; Some(custom_vocoder_fs_path) } @@ -141,17 +141,17 @@ async fn process_job_with_cleanup( let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str()); let tts_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash); - maybe_download_file_from_bucket( - "synthesizer", - &tts_synthesizer_fs_path, - &tts_synthesizer_object_path, - &args.job_dependencies.buckets.private_bucket_client, - &mut job_progress_reporter, - "downloading synthesizer", - job.id.0, - &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, - None, - ).await?; + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "synthesizer", + final_filesystem_file_path: & tts_synthesizer_fs_path, + bucket_object_path: &tts_synthesizer_object_path, + bucket_client: &args.job_dependencies.buckets.private_bucket_client, + job_progress_reporter: &mut job_progress_reporter, + job_progress_update_description: "downloading synthesizer", + job_id: job.id.0, + scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + maybe_existing_file_minimum_size_required: None, + }).await?; tts_synthesizer_fs_path }; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index 5789d3a56b..15ee5a53f8 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -22,7 +22,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::vits::vits_inference_command::{Device, VitsInferenceArgs}; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; /// Text starting with this will be treated as a test request. /// This allows the request to bypass the model cache and query the latest TTS model. @@ -70,17 +70,17 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result { + pub name_or_description_of_file: &'a str, + pub final_filesystem_file_path: &'a Path, + pub bucket_object_path: &'a Path, + pub bucket_client: &'a BucketClient, + pub job_progress_reporter: &'a mut Box, + pub job_progress_update_description: &'a str, + pub job_id: i64, + pub scoped_tempdir_creator: &'a ScopedTempDirCreator, + pub maybe_existing_file_minimum_size_required: Option, +} + pub async fn maybe_download_file_from_bucket( - name_or_description_of_file: &str, - final_filesystem_file_path: &Path, - bucket_object_path: &Path, - bucket_client: &BucketClient, - job_progress_reporter: &mut Box, - job_progress_update_description: &str, - job_id: i64, - scoped_tempdir_creator: &ScopedTempDirCreator, - maybe_existing_file_minimum_size_required: Option, + args: MaybeDownloadArgs<'_> ) -> Result<(), ProcessSingleJobError> { - if final_filesystem_file_path.exists() { + if args.final_filesystem_file_path.exists() { // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity let mut existing_file_is_valid = true; - if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required { + if let Some(existing_file_minimum_size_required) = args.maybe_existing_file_minimum_size_required { // NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file // size, but the Rust system may report a non-zero (but small) number of bytes. This should // later be investigated and the heuristic simplified. The intent here is merely to make sure @@ -40,11 +44,11 @@ pub async fn maybe_download_file_from_bucket( // -rw-r--r-- 1 root root 0 Nov 30 00:52 VM:1dzepsnwzbkc // -rw-r--r-- 1 root root 0 Nov 30 00:29 VM:7c2df5a36qjb // - let size = file_size(final_filesystem_file_path) + let size = file_size(args.final_filesystem_file_path) .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; - info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}", - &final_filesystem_file_path); + info!("{} exists at path {:?} ; file size = {size}", + &args.name_or_description_of_file,&args.final_filesystem_file_path); if size < existing_file_minimum_size_required { existing_file_is_valid = false; @@ -55,46 +59,46 @@ pub async fn maybe_download_file_from_bucket( return Ok(()) } } else { - warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); + warn!("{} does not exist at path: {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); } - job_progress_reporter.log_status(job_progress_update_description) + args.job_progress_reporter.log_status(args.job_progress_update_description) .map_err(|e| ProcessSingleJobError::Other(e))?; // NB: Download to temp directory to stop concurrent writes and race conditions from other // workers writing to a shared volume. - let temp_dir = format!("temp_download_{}", job_id); + let temp_dir = format!("temp_download_{}", args.job_id); // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem. - let temp_dir = scoped_tempdir_creator.new_tempdir(&temp_dir) + let temp_dir = args.scoped_tempdir_creator.new_tempdir(&temp_dir) .map_err(|e| ProcessSingleJobError::from_io_error(e))?; let temp_path = temp_dir.path().join("download.part"); - info!("Downloading {} from bucket path: {:?}", name_or_description_of_file, &bucket_object_path); + info!("Downloading {} from bucket path: {:?}", args.name_or_description_of_file, &args.bucket_object_path); - bucket_client.download_file_to_disk(&bucket_object_path, &temp_path) + args.bucket_client.download_file_to_disk(&args.bucket_object_path, &temp_path) .await .map_err(|e| { safe_delete_temp_directory(&temp_dir); ProcessSingleJobError::Other(e) })?; - info!("Downloaded {} from bucket!", name_or_description_of_file); + info!("Downloaded {} from bucket!", args.name_or_description_of_file); info!("Renaming {} temp file from {:?} to {:?}!", - name_or_description_of_file, &temp_path, &final_filesystem_file_path); + args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path); - rename_across_devices(&temp_path, &final_filesystem_file_path) + rename_across_devices(&temp_path, &args.final_filesystem_file_path) .map_err(|err| { error!("could not rename on disk: {:?}", err); safe_delete_temp_directory(&temp_dir); ProcessSingleJobError::from_io_error(err) })?; - info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path); + info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path); safe_delete_temp_directory(&temp_dir); Ok(()) -} \ No newline at end of file +}