download file size check and logging

crates/lib/files/filesys/src/file_size.rs

@@ -7,6 +7,7 @@ use errors::{anyhow, AnyhowResult};
 pub fn file_size<P: AsRef<Path>>(filename: P) -> AnyhowResult<u64> {
   let metadata = std::fs::metadata(filename.as_ref())?;
 
+  // TODO(bt,2023-11-29): Return io::Result and errors here.
   if !metadata.is_file() {
     Err(anyhow!("path {:?} is not a file", filename.as_ref()))
   } else if metadata.is_symlink() {

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs

@@ -103,6 +103,7 @@ pub async fn download_audio_file(
     "downloading",
     job.id.0,
     &temp_dir_creator,
+    None,
   ).await?;
 
   Ok(AudioFile {

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs

@@ -74,6 +74,7 @@ pub async fn download_image_file(
     "downloading",
     job.id.0,
     &temp_dir_creator,
+    None,
   ).await?;
 
   Ok(ImageFile {

crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/download_static_dependencies.rs

@@ -22,7 +22,6 @@ pub async fn download_static_dependencies<'a>(
   mut job_progress_reporter: &mut Box<dyn JobProgressReporter>,
 ) -> Result<StaticDependencies, ProcessSingleJobError> {
 
-
   // ==================== CONFIRM OR DOWNLOAD WAVEGLOW VOCODER MODEL ==================== //
 
   let waveglow_vocoder_model_fs_path = {
@@ -40,6 +39,7 @@ pub async fn download_static_dependencies<'a>(
       "downloading vocoder (1 of 3)",
       job.id.0,
       &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
+      Some(1000),
     ).await?;
 
     waveglow_vocoder_model_fs_path
@@ -62,6 +62,7 @@ pub async fn download_static_dependencies<'a>(
       "downloading vocoder (2 of 3)",
       job.id.0,
       &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
+      Some(1000),
     ).await?;
 
     hifigan_vocoder_model_fs_path
@@ -84,6 +85,7 @@ pub async fn download_static_dependencies<'a>(
       "downloading vocoder (3 of 3)",
       job.id.0,
       &job_dependencies.fs.scoped_temp_dir_creator_for_long_lived_downloads,
+      Some(1000),
     ).await?;
 
     hifigan_superres_vocoder_model_fs_path

crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs

@@ -128,6 +128,7 @@ async fn process_job_with_cleanup(
         "downloading user vocoder",
         job.id.0,
         &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+        None,
       ).await?;
 
       Some(custom_vocoder_fs_path)
@@ -149,6 +150,7 @@ async fn process_job_with_cleanup(
       "downloading synthesizer",
       job.id.0,
       &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      None,
     ).await?;
 
     tts_synthesizer_fs_path

crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs

@@ -79,6 +79,7 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result<JobSuccessResul
       "downloading synthesizer",
       job.id.0,
       &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      None,
     ).await?;
 
     vits_traced_synthesizer_fs_path

crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs

@@ -96,6 +96,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
       "downloading rvc (v2) model",
       job.id.0,
       &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      None,
     ).await?;
 
     fs_path
@@ -151,6 +152,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
         "downloading rvc (v2) model index",
         job.id.0,
         &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+        None,
       ).await?;
 
       Some(fs_path)
@@ -194,6 +196,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
       "downloading",
       job.id.0,
       &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+      None,
     ).await?;
 
     original_media_upload_fs_path

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs

@@ -85,6 +85,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
       "downloading so-vits-svc model",
       job.id.0,
       &args.job_dependencies.fs.scoped_temp_dir_creator_for_short_lived_downloads,
+      None,
     ).await?;
 
     so_vits_svc_fs_path
@@ -127,6 +128,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
       "downloading",
       job.id.0,
       &args.job_dependencies.fs.scoped_temp_dir_creator_for_work,
+      None,
     ).await?;
 
     original_media_upload_fs_path

crates/service/job/inference_job/src/util/maybe_download_file_from_bucket.rs

@@ -4,6 +4,7 @@ use log::{error, info, warn};
 
 use cloud_storage::bucket_client::BucketClient;
 use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
+use filesys::file_size::file_size;
 use filesys::rename_across_devices::rename_across_devices;
 use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
 
@@ -21,14 +22,41 @@ pub async fn maybe_download_file_from_bucket(
   job_progress_update_description: &str,
   job_id: i64,
   scoped_tempdir_creator: &ScopedTempDirCreator,
+  maybe_existing_file_minimum_size_required: Option<u64>,
 ) -> Result<(), ProcessSingleJobError> {
 
   if final_filesystem_file_path.exists() {
-    // TODO(bt, 2022-07-15): Check signature of file
-    return Ok(())
-  }
+    // TODO(bt, 2022-07-15): Check signature of file as best proof of file validity
+    let mut existing_file_is_valid = true;
 
-  warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path);
+    if let Some(existing_file_minimum_size_required) = maybe_existing_file_minimum_size_required {
+      // NB: Sometimes the downloader incompletely downloads files. Typically this is a zero file
+      // size, but the Rust system may report a non-zero (but small) number of bytes. This should
+      // later be investigated and the heuristic simplified. The intent here is merely to make sure
+      // we don't consider these okay:
+      //
+      //   -rw-r--r-- 1 root root 55824433 Nov 30 00:12 VM:012rkwsv91zb
+      //   -rw-r--r-- 1 root root 55823149 Nov 30 00:13 VM:79etbx4fdksv
+      //   -rw-r--r-- 1 root root        0 Nov 30 00:52 VM:1dzepsnwzbkc
+      //   -rw-r--r-- 1 root root        0 Nov 30 00:29 VM:7c2df5a36qjb
+      //
+      let size = file_size(final_filesystem_file_path)
+          .map_err(|err| ProcessSingleJobError::from_anyhow_error(err))?;
+
+      info!("{name_or_description_of_file} exists at path {:?} ; file size = {size}",
+        &final_filesystem_file_path);
+
+      if size < existing_file_minimum_size_required {
+        existing_file_is_valid = false;
+      }
+    }
+
+    if existing_file_is_valid {
+      return Ok(())
+    }
+  } else {
+    warn!("{} does not exist at path: {:?}", name_or_description_of_file, &final_filesystem_file_path);
+  }
 
   job_progress_reporter.log_status(job_progress_update_description)
       .map_err(|e| ProcessSingleJobError::Other(e))?;