diff --git a/crates/lib/files/filesys/src/file_size.rs b/crates/lib/files/filesys/src/file_size.rs index 89dcd79416..c64723f256 100644 --- a/crates/lib/files/filesys/src/file_size.rs +++ b/crates/lib/files/filesys/src/file_size.rs @@ -7,6 +7,7 @@ use errors::{anyhow, AnyhowResult}; pub fn file_size>(filename: P) -> AnyhowResult { let metadata = std::fs::metadata(filename.as_ref())?; + // TODO(bt,2023-11-29): Return io::Result and errors here. if !metadata.is_file() { Err(anyhow!("path {:?} is not a file", filename.as_ref())) } else if metadata.is_symlink() { diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index 6c0cb689a8..e810dc46d4 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -103,6 +103,7 @@ pub async fn download_audio_file( "downloading", job.id.0, &temp_dir_creator, + None, ).await?; Ok(AudioFile { diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index 506580bfe0..3ad2e15ab9 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -74,6 +74,7 @@ pub async fn download_image_file( "downloading", job.id.0, &temp_dir_creator, + None, ).await?; Ok(ImageFile { diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs index f0b3d2a0cd..60a43fe7a6 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs @@ -22,7 +22,6 @@ pub async fn download_static_dependencies<'a>( mut job_progress_reporter: &mut Box, ) -> Result { - // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== // let waveglow_vocoder_model_fs_path = { @@ -40,6 +39,7 @@ pub async fn download_static_dependencies<'a>( "downloading vocoder (1 of 3)", job.id.0, &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + Some(1000), ).await?; waveglow_vocoder_model_fs_path @@ -62,6 +62,7 @@ pub async fn download_static_dependencies<'a>( "downloading vocoder (2 of 3)", job.id.0, &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + Some(1000), ).await?; hifigan_vocoder_model_fs_path @@ -84,6 +85,7 @@ pub async fn download_static_dependencies<'a>( "downloading vocoder (3 of 3)", job.id.0, &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads, + Some(1000), ).await?; hifigan_superres_vocoder_model_fs_path diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 7c088abd9d..f5fbbf6457 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -128,6 +128,7 @@ async fn process_job_with_cleanup( "downloading user vocoder", job.id.0, &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + None, ).await?; Some(custom_vocoder_fs_path) @@ -149,6 +150,7 @@ async fn process_job_with_cleanup( "downloading synthesizer", job.id.0, &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads, + None, ).await?; tts_synthesizer_fs_path diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index d42cff9da9..5789d3a56b 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -79,6 +79,7 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result, ) -> Result<(), ProcessSingleJobError> { if final_filesystem_file_path.exists() { - // TODO(bt, 2022-07-15): Check signature of file - return Ok(()) - } + // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity + let mut existing_file_is_valid = true; - warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); + if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required { + // NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file + // size, but the Rust system may report a non-zero (but small) number of bytes. This should + // later be investigated and the heuristic simplified. The intent here is merely to make sure + // we don't consider these okay: + // + // -rw-r--r-- 1 root root 55824433 Nov 30 00:12 VM:012rkwsv91zb + // -rw-r--r-- 1 root root 55823149 Nov 30 00:13 VM:79etbx4fdksv + // -rw-r--r-- 1 root root 0 Nov 30 00:52 VM:1dzepsnwzbkc + // -rw-r--r-- 1 root root 0 Nov 30 00:29 VM:7c2df5a36qjb + // + let size = file_size(final_filesystem_file_path) + .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?; + + info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}", + &final_filesystem_file_path); + + if size < existing_file_minimum_size_required { + existing_file_is_valid = false; + } + } + + if existing_file_is_valid { + return Ok(()) + } + } else { + warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path); + } job_progress_reporter.log_status(job_progress_update_description) .map_err(|e| ProcessSingleJobError::Other(e))?;