struct args for downloader (safer)

This commit is contained in:
Brandon Thomas committed 2023-11-29 20:46:33 -05:00
1 parent 9e0772141a
commit 003f0cdeea
9 files changed
+178 -173

No files matched your search

@@ -3,6 +3,7 @@ use std::path::Path;
/// Rename a file.
/// Ordinary rename will fail in Linux if it is across physical devices.
/// This function will perform a copy followed by delete in that case.
/// In both cases, this function will overwrite the destination if it already exists.
pub fn rename_across_devices<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> std::io::Result<()> {
let result = std::fs::rename(&from, &to);
@@ -16,7 +16,7 @@ use mysql_queries::queries::voice_conversion::results::get_voice_conversion_resu
use tokens::tokens::media_uploads::MediaUploadToken;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
pub struct AudioFile {
@@ -94,17 +94,17 @@ pub async fn download_audio_file(
let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin");
maybe_download_file_from_bucket(
"audio file",
&downloaded_filesystem_path,
&bucket_object_path,
public_bucket_client,
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "audio file",
final_filesystem_file_path: &downloaded_filesystem_path,
bucket_object_path: &bucket_object_path,
bucket_client: public_bucket_client,
job_progress_reporter,
"downloading",
job.id.0,
&temp_dir_creator,
None,
).await?;
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &temp_dir_creator,
maybe_existing_file_minimum_size_required: None,
}).await?;
Ok(AudioFile {
filesystem_path: downloaded_filesystem_path,
@@ -14,7 +14,7 @@ use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_m
use tokens::tokens::media_uploads::MediaUploadToken;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
pub struct ImageFile {
@@ -65,17 +65,17 @@ pub async fn download_image_file(
let downloaded_filesystem_path = work_temp_dir.path().join("image.bin");
maybe_download_file_from_bucket(
"image file",
&downloaded_filesystem_path,
&bucket_object_path,
public_bucket_client,
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "image file",
final_filesystem_file_path: &downloaded_filesystem_path,
bucket_object_path: &bucket_object_path,
bucket_client: public_bucket_client,
job_progress_reporter,
"downloading",
job.id.0,
&temp_dir_creator,
None,
).await?;
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &temp_dir_creator,
maybe_existing_file_minimum_size_required: None,
}).await?;
Ok(ImageFile {
filesystem_path: downloaded_filesystem_path,
@@ -6,7 +6,7 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies;
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
pub struct StaticDependencies {
pub waveglow_vocoder_model_fs_path : PathBuf,
@@ -30,17 +30,17 @@ pub async fn download_static_dependencies<'a>(
let waveglow_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&waveglow_vocoder_model_filename);
let waveglow_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename);
maybe_download_file_from_bucket(
"waveglow vocoder model",
&waveglow_vocoder_model_fs_path,
&waveglow_vocoder_model_object_path,
&job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading vocoder (1 of 3)",
job.id.0,
&job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
Some(1000),
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "waveglow vocoder model",
final_filesystem_file_path: &waveglow_vocoder_model_fs_path,
bucket_object_path: &waveglow_vocoder_model_object_path,
bucket_client: &job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading vocoder (1 of 3)",
job_id: job.id.0,
scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
maybe_existing_file_minimum_size_required: Some(1000),
}).await?;
waveglow_vocoder_model_fs_path
};
@@ -53,17 +53,17 @@ pub async fn download_static_dependencies<'a>(
let hifigan_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_vocoder_model_filename);
let hifigan_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename);
maybe_download_file_from_bucket(
"hifigan vocoder model",
&hifigan_vocoder_model_fs_path,
&hifigan_vocoder_model_object_path,
&job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading vocoder (2 of 3)",
job.id.0,
&job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
Some(1000),
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "hifigan vocoder model",
final_filesystem_file_path: &hifigan_vocoder_model_fs_path,
bucket_object_path: &hifigan_vocoder_model_object_path,
bucket_client: &job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading vocoder (2 of 3)",
job_id: job.id.0,
scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
maybe_existing_file_minimum_size_required: Some(1000),
}).await?;
hifigan_vocoder_model_fs_path
};
@@ -76,17 +76,17 @@ pub async fn download_static_dependencies<'a>(
let hifigan_superres_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_superres_vocoder_model_filename);
let hifigan_superres_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename);
maybe_download_file_from_bucket(
"hifigan superres vocoder model",
&hifigan_superres_vocoder_model_fs_path,
&hifigan_superres_vocoder_model_object_path,
&job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading vocoder (3 of 3)",
job.id.0,
&job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
Some(1000),
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "hifigan superres vocoder model",
final_filesystem_file_path: &hifigan_superres_vocoder_model_fs_path,
bucket_object_path: &hifigan_superres_vocoder_model_object_path,
bucket_client: &job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading vocoder (3 of 3)",
job_id: job.id.0,
scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
maybe_existing_file_minimum_size_required: Some(1000),
}).await?;
hifigan_superres_vocoder_model_fs_path
};
@@ -29,7 +29,7 @@ use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_s
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor};
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption;
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
/// Text starting with this will be treated as a test request.
/// This allows the request to bypass the model cache and query the latest TTS model.
@@ -119,17 +119,17 @@ async fn process_job_with_cleanup(
let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token);
let custom_vocoder_object_path = args.job_dependencies.buckets.bucket_path_unifier.vocoder_path(&vocoder.vocoder_private_bucket_hash);
maybe_download_file_from_bucket(
"custom vocoder",
&custom_vocoder_fs_path,
&custom_vocoder_object_path,
&args.job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading user vocoder",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "custom vocoder",
final_filesystem_file_path: &custom_vocoder_fs_path,
bucket_object_path: &custom_vocoder_object_path,
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading user vocoder",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
maybe_existing_file_minimum_size_required: None,
}).await?;
Some(custom_vocoder_fs_path)
}
@@ -141,17 +141,17 @@ async fn process_job_with_cleanup(
let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str());
let tts_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash);
maybe_download_file_from_bucket(
"synthesizer",
&tts_synthesizer_fs_path,
&tts_synthesizer_object_path,
&args.job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading synthesizer",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "synthesizer",
final_filesystem_file_path: & tts_synthesizer_fs_path,
bucket_object_path: &tts_synthesizer_object_path,
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading synthesizer",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
maybe_existing_file_minimum_size_required: None,
}).await?;
tts_synthesizer_fs_path
};
@@ -22,7 +22,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::tts::vits::vits_inference_command::{Device, VitsInferenceArgs};
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
/// Text starting with this will be treated as a test request.
/// This allows the request to bypass the model cache and query the latest TTS model.
@@ -70,17 +70,17 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result<JobSuccessResul
// In the future we may need to "repair" broken models.
let vits_traced_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_traced_synthesizer_path(&tts_model.private_bucket_hash);
maybe_download_file_from_bucket(
"vits traced tts model",
&vits_traced_synthesizer_fs_path,
&vits_traced_synthesizer_object_path,
&args.job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading synthesizer",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "vits traced tts model",
final_filesystem_file_path: &vits_traced_synthesizer_fs_path,
bucket_object_path: &vits_traced_synthesizer_object_path,
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading synthesizer",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
maybe_existing_file_minimum_size_required: None,
}).await?;
vits_traced_synthesizer_fs_path
};
@@ -27,7 +27,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::InferenceArgs;
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
const BUCKET_FILE_PREFIX : &str = "fakeyou_rvc_";
const BUCKET_FILE_EXTENSION : &str = ".wav";
@@ -87,17 +87,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
let model_object_path = args.job_dependencies.buckets.bucket_path_unifier.rvc_v2_model_path(&vc_model.private_bucket_hash);
maybe_download_file_from_bucket(
"rvc (v2) model",
&fs_path,
&model_object_path,
&args.job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading rvc (v2) model",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "rvc (v2) model",
final_filesystem_file_path: &fs_path,
bucket_object_path: &model_object_path,
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading rvc (v2) model",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
maybe_existing_file_minimum_size_required: None,
}).await?;
fs_path
};
@@ -143,17 +143,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
let model_index_object_path = args.job_dependencies.buckets.bucket_path_unifier.rvc_v2_model_index_path(&vc_model.private_bucket_hash);
maybe_download_file_from_bucket(
"rvc (v2) model index",
&fs_path,
&model_index_object_path,
&args.job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading rvc (v2) model index",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "rvc (v2) model index",
final_filesystem_file_path: &fs_path,
bucket_object_path: &model_index_object_path,
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading rvc (v2) model index",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
maybe_existing_file_minimum_size_required: None,
}).await?;
Some(fs_path)
};
@@ -187,17 +187,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
info!("Downloading media from bucket path: {:?}", &bucket_object_path);
maybe_download_file_from_bucket(
"media upload (original file)",
&original_media_upload_fs_path,
&bucket_object_path,
&args.job_dependencies.buckets.public_bucket_client,
&mut job_progress_reporter,
"downloading",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "media upload (original file)",
final_filesystem_file_path: &original_media_upload_fs_path,
bucket_object_path: &bucket_object_path,
bucket_client: &args.job_dependencies.buckets.public_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
maybe_existing_file_minimum_size_required: None,
}).await?;
original_media_upload_fs_path
};
@@ -26,7 +26,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::{Device, InferenceArgs};
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
const BUCKET_FILE_PREFIX : &str = "fakeyou_svc_";
const BUCKET_FILE_EXTENSION : &str = ".wav";
@@ -76,17 +76,17 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
let so_vits_svc_model_object_path = args.job_dependencies.buckets.bucket_path_unifier.so_vits_svc_model_path(&vc_model.private_bucket_hash);
maybe_download_file_from_bucket(
"so-vits-svc model",
&so_vits_svc_fs_path,
&so_vits_svc_model_object_path,
&args.job_dependencies.buckets.private_bucket_client,
&mut job_progress_reporter,
"downloading so-vits-svc model",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "so-vits-svc model",
final_filesystem_file_path: &so_vits_svc_fs_path,
bucket_object_path: &so_vits_svc_model_object_path,
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading so-vits-svc model",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
maybe_existing_file_minimum_size_required: None,
}).await?;
so_vits_svc_fs_path
};
@@ -119,17 +119,17 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
info!("Downloading media to bucket path: {:?}", &bucket_object_path);
maybe_download_file_from_bucket(
"media upload (original file)",
&original_media_upload_fs_path,
&bucket_object_path,
&args.job_dependencies.buckets.public_bucket_client,
&mut job_progress_reporter,
"downloading",
job.id.0,
&args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
None,
).await?;
maybe_download_file_from_bucket(MaybeDownloadArgs {
name_or_description_of_file: "media upload (original file)",
final_filesystem_file_path: &original_media_upload_fs_path,
bucket_object_path: &bucket_object_path,
bucket_client: &args.job_dependencies.buckets.public_bucket_client,
job_progress_reporter: &mut job_progress_reporter,
job_progress_update_description: "downloading",
job_id: job.id.0,
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
maybe_existing_file_minimum_size_required: None,
}).await?;
original_media_upload_fs_path
};
@@ -13,23 +13,27 @@ use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
// TODO(bt, 2022-07-15): Make a concrete type for bucket paths
pub struct MaybeDownloadArgs<'a> {
pub name_or_description_of_file: &'a str,
pub final_filesystem_file_path: &'a Path,
pub bucket_object_path: &'a Path,
pub bucket_client: &'a BucketClient,
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
pub job_progress_update_description: &'a str,
pub job_id: i64,
pub scoped_tempdir_creator: &'a ScopedTempDirCreator,
pub maybe_existing_file_minimum_size_required: Option<u64>,
}
pub async fn maybe_download_file_from_bucket(
name_or_description_of_file: &str,
final_filesystem_file_path: &Path,
bucket_object_path: &Path,
bucket_client: &BucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job_progress_update_description: &str,
job_id: i64,
scoped_tempdir_creator: &ScopedTempDirCreator,
maybe_existing_file_minimum_size_required: Option<u64>,
args: MaybeDownloadArgs<'_>
) -> Result<(), ProcessSingleJobError> {
if final_filesystem_file_path.exists() {
if args.final_filesystem_file_path.exists() {
// TODO(bt, 2022-07-15): Check signature of file as best proof of file validity
let mut existing_file_is_valid = true;
if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required {
if let Some(existing_file_minimum_size_required) = args.maybe_existing_file_minimum_size_required {
// NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file
// size, but the Rust system may report a non-zero (but small) number of bytes. This should
// later be investigated and the heuristic simplified. The intent here is merely to make sure
@@ -40,11 +44,11 @@ pub async fn maybe_download_file_from_bucket(
// -rw-r--r-- 1 root root 0 Nov 30 00:52 VM:1dzepsnwzbkc
// -rw-r--r-- 1 root root 0 Nov 30 00:29 VM:7c2df5a36qjb
//
let size = file_size(final_filesystem_file_path)
let size = file_size(args.final_filesystem_file_path)
.map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}",
&final_filesystem_file_path);
info!("{} exists at path {:?} ; file size = {size}",
&args.name_or_description_of_file,&args.final_filesystem_file_path);
if size < existing_file_minimum_size_required {
existing_file_is_valid = false;
@@ -55,46 +59,46 @@ pub async fn maybe_download_file_from_bucket(
return Ok(())
}
} else {
warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path);
warn!("{} does not exist at path: {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path);
}
job_progress_reporter.log_status(job_progress_update_description)
args.job_progress_reporter.log_status(args.job_progress_update_description)
.map_err(|e| ProcessSingleJobError::Other(e))?;
// NB: Download to temp directory to stop concurrent writes and race conditions from other
// workers writing to a shared volume.
let temp_dir = format!("temp_download_{}", job_id);
let temp_dir = format!("temp_download_{}", args.job_id);
// NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
let temp_dir = scoped_tempdir_creator.new_tempdir(&temp_dir)
let temp_dir = args.scoped_tempdir_creator.new_tempdir(&temp_dir)
.map_err(|e| ProcessSingleJobError::from_io_error(e))?;
let temp_path = temp_dir.path().join("download.part");
info!("Downloading {} from bucket path: {:?}", name_or_description_of_file, &bucket_object_path);
info!("Downloading {} from bucket path: {:?}", args.name_or_description_of_file, &args.bucket_object_path);
bucket_client.download_file_to_disk(&bucket_object_path, &temp_path)
args.bucket_client.download_file_to_disk(&args.bucket_object_path, &temp_path)
.await
.map_err(|e| {
safe_delete_temp_directory(&temp_dir);
ProcessSingleJobError::Other(e)
})?;
info!("Downloaded {} from bucket!", name_or_description_of_file);
info!("Downloaded {} from bucket!", args.name_or_description_of_file);
info!("Renaming {} temp file from {:?} to {:?}!",
name_or_description_of_file, &temp_path, &final_filesystem_file_path);
args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path);
rename_across_devices(&temp_path, &final_filesystem_file_path)
rename_across_devices(&temp_path, &args.final_filesystem_file_path)
.map_err(|err| {
error!("could not rename on disk: {:?}", err);
safe_delete_temp_directory(&temp_dir);
ProcessSingleJobError::from_io_error(err)
})?;
info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path);
info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path);
safe_delete_temp_directory(&temp_dir);
Ok(())
}
}