mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-17 07:04:36 +00:00
struct args for downloader (safer)
This commit is contained in:
1 parent
9e0772141a
commit
003f0cdeea
9 files changed
+178
-173
No files matched your search
@@ -3,6 +3,7 @@ use std::path::Path;
|
||||
/// Rename a file.
|
||||
/// Ordinary rename will fail in Linux if it is across physical devices.
|
||||
/// This function will perform a copy followed by delete in that case.
|
||||
/// In both cases, this function will overwrite the destination if it already exists.
|
||||
pub fn rename_across_devices<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> std::io::Result<()> {
|
||||
let result = std::fs::rename(&from, &to);
|
||||
|
||||
|
||||
+11
-11
@@ -16,7 +16,7 @@ use mysql_queries::queries::voice_conversion::results::get_voice_conversion_resu
|
||||
use tokens::tokens::media_uploads::MediaUploadToken;
|
||||
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
|
||||
|
||||
pub struct AudioFile {
|
||||
@@ -94,17 +94,17 @@ pub async fn download_audio_file(
|
||||
|
||||
let downloaded_filesystem_path = work_temp_dir.path().join("audio.bin");
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"audio file",
|
||||
&downloaded_filesystem_path,
|
||||
&bucket_object_path,
|
||||
public_bucket_client,
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "audio file",
|
||||
final_filesystem_file_path: &downloaded_filesystem_path,
|
||||
bucket_object_path: &bucket_object_path,
|
||||
bucket_client: public_bucket_client,
|
||||
job_progress_reporter,
|
||||
"downloading",
|
||||
job.id.0,
|
||||
&temp_dir_creator,
|
||||
None,
|
||||
).await?;
|
||||
job_progress_update_description: "downloading",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &temp_dir_creator,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
Ok(AudioFile {
|
||||
filesystem_path: downloaded_filesystem_path,
|
||||
|
||||
+11
-11
@@ -14,7 +14,7 @@ use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_m
|
||||
use tokens::tokens::media_uploads::MediaUploadToken;
|
||||
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
|
||||
|
||||
pub struct ImageFile {
|
||||
@@ -65,17 +65,17 @@ pub async fn download_image_file(
|
||||
|
||||
let downloaded_filesystem_path = work_temp_dir.path().join("image.bin");
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"image file",
|
||||
&downloaded_filesystem_path,
|
||||
&bucket_object_path,
|
||||
public_bucket_client,
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "image file",
|
||||
final_filesystem_file_path: &downloaded_filesystem_path,
|
||||
bucket_object_path: &bucket_object_path,
|
||||
bucket_client: public_bucket_client,
|
||||
job_progress_reporter,
|
||||
"downloading",
|
||||
job.id.0,
|
||||
&temp_dir_creator,
|
||||
None,
|
||||
).await?;
|
||||
job_progress_update_description: "downloading",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &temp_dir_creator,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
Ok(ImageFile {
|
||||
filesystem_path: downloaded_filesystem_path,
|
||||
|
||||
+34
-34
@@ -6,7 +6,7 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
|
||||
pub struct StaticDependencies {
|
||||
pub waveglow_vocoder_model_fs_path : PathBuf,
|
||||
@@ -30,17 +30,17 @@ pub async fn download_static_dependencies<'a>(
|
||||
let waveglow_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&waveglow_vocoder_model_filename);
|
||||
let waveglow_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&waveglow_vocoder_model_filename);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"waveglow vocoder model",
|
||||
&waveglow_vocoder_model_fs_path,
|
||||
&waveglow_vocoder_model_object_path,
|
||||
&job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading vocoder (1 of 3)",
|
||||
job.id.0,
|
||||
&job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
|
||||
Some(1000),
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "waveglow vocoder model",
|
||||
final_filesystem_file_path: &waveglow_vocoder_model_fs_path,
|
||||
bucket_object_path: &waveglow_vocoder_model_object_path,
|
||||
bucket_client: &job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading vocoder (1 of 3)",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: Some(1000),
|
||||
}).await?;
|
||||
|
||||
waveglow_vocoder_model_fs_path
|
||||
};
|
||||
@@ -53,17 +53,17 @@ pub async fn download_static_dependencies<'a>(
|
||||
let hifigan_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_vocoder_model_filename);
|
||||
let hifigan_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_vocoder_model_filename);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"hifigan vocoder model",
|
||||
&hifigan_vocoder_model_fs_path,
|
||||
&hifigan_vocoder_model_object_path,
|
||||
&job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading vocoder (2 of 3)",
|
||||
job.id.0,
|
||||
&job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
|
||||
Some(1000),
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "hifigan vocoder model",
|
||||
final_filesystem_file_path: &hifigan_vocoder_model_fs_path,
|
||||
bucket_object_path: &hifigan_vocoder_model_object_path,
|
||||
bucket_client: &job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading vocoder (2 of 3)",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: Some(1000),
|
||||
}).await?;
|
||||
|
||||
hifigan_vocoder_model_fs_path
|
||||
};
|
||||
@@ -76,17 +76,17 @@ pub async fn download_static_dependencies<'a>(
|
||||
let hifigan_superres_vocoder_model_fs_path = job_dependencies.fs.semi_persistent_cache.tts_pretrained_vocoder_model_path(&hifigan_superres_vocoder_model_filename);
|
||||
let hifigan_superres_vocoder_model_object_path = job_dependencies.buckets.bucket_path_unifier.tts_pretrained_vocoders_path(&hifigan_superres_vocoder_model_filename);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"hifigan superres vocoder model",
|
||||
&hifigan_superres_vocoder_model_fs_path,
|
||||
&hifigan_superres_vocoder_model_object_path,
|
||||
&job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading vocoder (3 of 3)",
|
||||
job.id.0,
|
||||
&job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
|
||||
Some(1000),
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "hifigan superres vocoder model",
|
||||
final_filesystem_file_path: &hifigan_superres_vocoder_model_fs_path,
|
||||
bucket_object_path: &hifigan_superres_vocoder_model_object_path,
|
||||
bucket_client: &job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading vocoder (3 of 3)",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: Some(1000),
|
||||
}).await?;
|
||||
|
||||
hifigan_superres_vocoder_model_fs_path
|
||||
};
|
||||
|
||||
+23
-23
@@ -29,7 +29,7 @@ use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_s
|
||||
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor};
|
||||
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
|
||||
/// Text starting with this will be treated as a test request.
|
||||
/// This allows the request to bypass the model cache and query the latest TTS model.
|
||||
@@ -119,17 +119,17 @@ async fn process_job_with_cleanup(
|
||||
let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token);
|
||||
let custom_vocoder_object_path = args.job_dependencies.buckets.bucket_path_unifier.vocoder_path(&vocoder.vocoder_private_bucket_hash);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"custom vocoder",
|
||||
&custom_vocoder_fs_path,
|
||||
&custom_vocoder_object_path,
|
||||
&args.job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading user vocoder",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "custom vocoder",
|
||||
final_filesystem_file_path: &custom_vocoder_fs_path,
|
||||
bucket_object_path: &custom_vocoder_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading user vocoder",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
Some(custom_vocoder_fs_path)
|
||||
}
|
||||
@@ -141,17 +141,17 @@ async fn process_job_with_cleanup(
|
||||
let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str());
|
||||
let tts_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"synthesizer",
|
||||
&tts_synthesizer_fs_path,
|
||||
&tts_synthesizer_object_path,
|
||||
&args.job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading synthesizer",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "synthesizer",
|
||||
final_filesystem_file_path: & tts_synthesizer_fs_path,
|
||||
bucket_object_path: &tts_synthesizer_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading synthesizer",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
tts_synthesizer_fs_path
|
||||
};
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::job::job_types::tts::vits::vits_inference_command::{Device, VitsInferenceArgs};
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
|
||||
/// Text starting with this will be treated as a test request.
|
||||
/// This allows the request to bypass the model cache and query the latest TTS model.
|
||||
@@ -70,17 +70,17 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result<JobSuccessResul
|
||||
// In the future we may need to "repair" broken models.
|
||||
let vits_traced_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_traced_synthesizer_path(&tts_model.private_bucket_hash);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"vits traced tts model",
|
||||
&vits_traced_synthesizer_fs_path,
|
||||
&vits_traced_synthesizer_object_path,
|
||||
&args.job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading synthesizer",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "vits traced tts model",
|
||||
final_filesystem_file_path: &vits_traced_synthesizer_fs_path,
|
||||
bucket_object_path: &vits_traced_synthesizer_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading synthesizer",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
vits_traced_synthesizer_fs_path
|
||||
};
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::InferenceArgs;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
|
||||
const BUCKET_FILE_PREFIX : &str = "fakeyou_rvc_";
|
||||
const BUCKET_FILE_EXTENSION : &str = ".wav";
|
||||
@@ -87,17 +87,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
|
||||
|
||||
let model_object_path = args.job_dependencies.buckets.bucket_path_unifier.rvc_v2_model_path(&vc_model.private_bucket_hash);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"rvc (v2) model",
|
||||
&fs_path,
|
||||
&model_object_path,
|
||||
&args.job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading rvc (v2) model",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "rvc (v2) model",
|
||||
final_filesystem_file_path: &fs_path,
|
||||
bucket_object_path: &model_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading rvc (v2) model",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
fs_path
|
||||
};
|
||||
@@ -143,17 +143,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
|
||||
|
||||
let model_index_object_path = args.job_dependencies.buckets.bucket_path_unifier.rvc_v2_model_index_path(&vc_model.private_bucket_hash);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"rvc (v2) model index",
|
||||
&fs_path,
|
||||
&model_index_object_path,
|
||||
&args.job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading rvc (v2) model index",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "rvc (v2) model index",
|
||||
final_filesystem_file_path: &fs_path,
|
||||
bucket_object_path: &model_index_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading rvc (v2) model index",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
Some(fs_path)
|
||||
};
|
||||
@@ -187,17 +187,17 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
|
||||
|
||||
info!("Downloading media from bucket path: {:?}", &bucket_object_path);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"media upload (original file)",
|
||||
&original_media_upload_fs_path,
|
||||
&bucket_object_path,
|
||||
&args.job_dependencies.buckets.public_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "media upload (original file)",
|
||||
final_filesystem_file_path: &original_media_upload_fs_path,
|
||||
bucket_object_path: &bucket_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.public_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
original_media_upload_fs_path
|
||||
};
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::{Device, InferenceArgs};
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
|
||||
use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
|
||||
|
||||
const BUCKET_FILE_PREFIX : &str = "fakeyou_svc_";
|
||||
const BUCKET_FILE_EXTENSION : &str = ".wav";
|
||||
@@ -76,17 +76,17 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
|
||||
|
||||
let so_vits_svc_model_object_path = args.job_dependencies.buckets.bucket_path_unifier.so_vits_svc_model_path(&vc_model.private_bucket_hash);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"so-vits-svc model",
|
||||
&so_vits_svc_fs_path,
|
||||
&so_vits_svc_model_object_path,
|
||||
&args.job_dependencies.buckets.private_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading so-vits-svc model",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "so-vits-svc model",
|
||||
final_filesystem_file_path: &so_vits_svc_fs_path,
|
||||
bucket_object_path: &so_vits_svc_model_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.private_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading so-vits-svc model",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
so_vits_svc_fs_path
|
||||
};
|
||||
@@ -119,17 +119,17 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
|
||||
|
||||
info!("Downloading media to bucket path: {:?}", &bucket_object_path);
|
||||
|
||||
maybe_download_file_from_bucket(
|
||||
"media upload (original file)",
|
||||
&original_media_upload_fs_path,
|
||||
&bucket_object_path,
|
||||
&args.job_dependencies.buckets.public_bucket_client,
|
||||
&mut job_progress_reporter,
|
||||
"downloading",
|
||||
job.id.0,
|
||||
&args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
|
||||
None,
|
||||
).await?;
|
||||
maybe_download_file_from_bucket(MaybeDownloadArgs {
|
||||
name_or_description_of_file: "media upload (original file)",
|
||||
final_filesystem_file_path: &original_media_upload_fs_path,
|
||||
bucket_object_path: &bucket_object_path,
|
||||
bucket_client: &args.job_dependencies.buckets.public_bucket_client,
|
||||
job_progress_reporter: &mut job_progress_reporter,
|
||||
job_progress_update_description: "downloading",
|
||||
job_id: job.id.0,
|
||||
scoped_tempdir_creator: &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
|
||||
maybe_existing_file_minimum_size_required: None,
|
||||
}).await?;
|
||||
|
||||
original_media_upload_fs_path
|
||||
};
|
||||
|
||||
@@ -13,23 +13,27 @@ use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
|
||||
|
||||
// TODO(bt, 2022-07-15): Make a concrete type for bucket paths
|
||||
|
||||
pub struct MaybeDownloadArgs<'a> {
|
||||
pub name_or_description_of_file: &'a str,
|
||||
pub final_filesystem_file_path: &'a Path,
|
||||
pub bucket_object_path: &'a Path,
|
||||
pub bucket_client: &'a BucketClient,
|
||||
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
|
||||
pub job_progress_update_description: &'a str,
|
||||
pub job_id: i64,
|
||||
pub scoped_tempdir_creator: &'a ScopedTempDirCreator,
|
||||
pub maybe_existing_file_minimum_size_required: Option<u64>,
|
||||
}
|
||||
|
||||
pub async fn maybe_download_file_from_bucket(
|
||||
name_or_description_of_file: &str,
|
||||
final_filesystem_file_path: &Path,
|
||||
bucket_object_path: &Path,
|
||||
bucket_client: &BucketClient,
|
||||
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
|
||||
job_progress_update_description: &str,
|
||||
job_id: i64,
|
||||
scoped_tempdir_creator: &ScopedTempDirCreator,
|
||||
maybe_existing_file_minimum_size_required: Option<u64>,
|
||||
args: MaybeDownloadArgs<'_>
|
||||
) -> Result<(), ProcessSingleJobError> {
|
||||
|
||||
if final_filesystem_file_path.exists() {
|
||||
if args.final_filesystem_file_path.exists() {
|
||||
// TODO(bt, 2022-07-15): Check signature of file as best proof of file validity
|
||||
let mut existing_file_is_valid = true;
|
||||
|
||||
if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required {
|
||||
if let Some(existing_file_minimum_size_required) = args.maybe_existing_file_minimum_size_required {
|
||||
// NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file
|
||||
// size, but the Rust system may report a non-zero (but small) number of bytes. This should
|
||||
// later be investigated and the heuristic simplified. The intent here is merely to make sure
|
||||
@@ -40,11 +44,11 @@ pub async fn maybe_download_file_from_bucket(
|
||||
// -rw-r--r-- 1 root root 0 Nov 30 00:52 VM:1dzepsnwzbkc
|
||||
// -rw-r--r-- 1 root root 0 Nov 30 00:29 VM:7c2df5a36qjb
|
||||
//
|
||||
let size = file_size(final_filesystem_file_path)
|
||||
let size = file_size(args.final_filesystem_file_path)
|
||||
.map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
|
||||
|
||||
info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}",
|
||||
&final_filesystem_file_path);
|
||||
info!("{} exists at path {:?} ; file size = {size}",
|
||||
&args.name_or_description_of_file,&args.final_filesystem_file_path);
|
||||
|
||||
if size < existing_file_minimum_size_required {
|
||||
existing_file_is_valid = false;
|
||||
@@ -55,46 +59,46 @@ pub async fn maybe_download_file_from_bucket(
|
||||
return Ok(())
|
||||
}
|
||||
} else {
|
||||
warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path);
|
||||
warn!("{} does not exist at path: {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path);
|
||||
}
|
||||
|
||||
job_progress_reporter.log_status(job_progress_update_description)
|
||||
args.job_progress_reporter.log_status(args.job_progress_update_description)
|
||||
.map_err(|e| ProcessSingleJobError::Other(e))?;
|
||||
|
||||
// NB: Download to temp directory to stop concurrent writes and race conditions from other
|
||||
// workers writing to a shared volume.
|
||||
let temp_dir = format!("temp_download_{}", job_id);
|
||||
let temp_dir = format!("temp_download_{}", args.job_id);
|
||||
|
||||
// NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem.
|
||||
let temp_dir = scoped_tempdir_creator.new_tempdir(&temp_dir)
|
||||
let temp_dir = args.scoped_tempdir_creator.new_tempdir(&temp_dir)
|
||||
.map_err(|e| ProcessSingleJobError::from_io_error(e))?;
|
||||
|
||||
let temp_path = temp_dir.path().join("download.part");
|
||||
|
||||
info!("Downloading {} from bucket path: {:?}", name_or_description_of_file, &bucket_object_path);
|
||||
info!("Downloading {} from bucket path: {:?}", args.name_or_description_of_file, &args.bucket_object_path);
|
||||
|
||||
bucket_client.download_file_to_disk(&bucket_object_path, &temp_path)
|
||||
args.bucket_client.download_file_to_disk(&args.bucket_object_path, &temp_path)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
safe_delete_temp_directory(&temp_dir);
|
||||
ProcessSingleJobError::Other(e)
|
||||
})?;
|
||||
|
||||
info!("Downloaded {} from bucket!", name_or_description_of_file);
|
||||
info!("Downloaded {} from bucket!", args.name_or_description_of_file);
|
||||
|
||||
info!("Renaming {} temp file from {:?} to {:?}!",
|
||||
name_or_description_of_file, &temp_path, &final_filesystem_file_path);
|
||||
args.name_or_description_of_file, &temp_path, &args.final_filesystem_file_path);
|
||||
|
||||
rename_across_devices(&temp_path, &final_filesystem_file_path)
|
||||
rename_across_devices(&temp_path, &args.final_filesystem_file_path)
|
||||
.map_err(|err| {
|
||||
error!("could not rename on disk: {:?}", err);
|
||||
safe_delete_temp_directory(&temp_dir);
|
||||
ProcessSingleJobError::from_io_error(err)
|
||||
})?;
|
||||
|
||||
info!("Finished downloading {} file to {:?}", name_or_description_of_file, &final_filesystem_file_path);
|
||||
info!("Finished downloading {} file to {:?}", args.name_or_description_of_file, &args.final_filesystem_file_path);
|
||||
|
||||
safe_delete_temp_directory(&temp_dir);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user