WIP generic-download-job

_migrations/2022-08-05-202017_generic_download_jobs/up.sql

@@ -39,7 +39,7 @@ CREATE TABLE generic_download_jobs (
   -- Assume the same URL can be uploaded more than once (in case of errors or
   -- people sharing models)
 
-  download_url VARCHAR(512) DEFAULT NULL,
+  download_url VARCHAR(512) NOT NULL,
 
   -- ========== CREATOR DETAILS AND PREFERENCES ==========
 

crates/lib/config/src/bad_urls.rs

@@ -1,36 +1,11 @@
 //! Probably should be in its own "static_config" crate.
 
-use url::{Host, Url};
+use crate::is_bad_download_url::is_bad_download_url;
 
 /// Reports if the URL is inappropriate for downloading as a TTS model, vocoder, etc.
 pub fn is_bad_tts_model_download_url(url: &str) -> anyhow::Result<bool> {
-
-  if url.trim().is_empty() {
-    // We can't use empty URLs.
-    return Ok(true);
-  }
-
-  let url = Url::parse(url)?;
-
-  match url.host() {
-    Some(Host::Domain(domain)) => {
-      let domain = domain.to_lowercase();
-
-      // TODO: If this list grows, store it in a static hashset
-      let bad_host = domain.contains("fb.watch")
-          || domain.contains("tiktok.com")
-          || domain.contains("vm.tiktok.com") // NB: This hostname is known for never disconnecting, which freezes the job
-          || domain.contains("youtu.be")
-          || domain.contains("youtube.com");
-
-      if bad_host {
-        return Ok(true)
-      }
-    }
-    _ => {},
-  }
-
-  Ok(false)
+  // TODO: Just use this.
+  is_bad_download_url(url)
 }
 
 #[cfg(test)]

crates/lib/config/src/is_bad_download_url.rs

@@ -0,0 +1,53 @@
+//! Probably should be in its own "static_config" crate.
+
+use url::{Host, Url};
+
+/// Reports if the URL is inappropriate for downloading as a TTS model, vocoder, etc.
+pub fn is_bad_download_url(url: &str) -> anyhow::Result<bool> {
+
+  if url.trim().is_empty() {
+    // We can't use empty URLs.
+    return Ok(true);
+  }
+
+  let url = Url::parse(url)?;
+
+  match url.host() {
+    Some(Host::Domain(domain)) => {
+      let domain = domain.to_lowercase();
+
+      // TODO: If this list grows, store it in a static hashset
+      let bad_host = domain.contains("fb.watch")
+          || domain.contains("tiktok.com")
+          || domain.contains("vm.tiktok.com") // NB: This hostname is known for never disconnecting, which freezes the job
+          || domain.contains("youtu.be")
+          || domain.contains("youtube.com");
+
+      if bad_host {
+        return Ok(true)
+      }
+    }
+    _ => {},
+  }
+
+  Ok(false)
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::is_bad_download_url::is_bad_download_url;
+
+  #[test]
+  fn bad_download_url() {
+    assert_eq!(is_bad_download_url("").unwrap(), true);
+    assert_eq!(is_bad_download_url("   ").unwrap(), true);
+    assert_eq!(is_bad_download_url("https://vm.tiktok.com/ZMNYjT7Xy/?k=1 ").unwrap(), true); // NB: We get lots of these
+    assert_eq!(is_bad_download_url("https://m.youtube.com/watch?v=HY-vzGBiAZo").unwrap(), true); // NB: We get lots of these
+  }
+
+  #[test]
+  fn good_tts_model_url() {
+    assert_eq!(is_bad_download_url("https://drive.google.com/file/d/1-1kEoX4HGCwJm4R9cZhSVByWmUoQVGVm/view").unwrap(), false);
+    assert_eq!(is_bad_download_url("https://drive.google.com/file/d/1SofQhvSkDY-vi_zuBfHivBbJo4CqhJeH/view?usp=sharing").unwrap(), false);
+  }
+}

crates/lib/config/src/lib.rs

@@ -1,3 +1,4 @@
 pub mod bad_urls;
 pub mod common_env;
+pub mod is_bad_download_url;
 pub mod shared_constants;

crates/lib/database_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs

@@ -20,7 +20,7 @@ pub struct AvailableDownloadJob {
   pub creator_set_visibility: EntityVisibility,
 
   pub download_type: GenericDownloadType,
-  pub download_url: Option<String>,
+  pub download_url: String,
   pub title: String,
 
   pub status: JobStatus,
@@ -110,7 +110,7 @@ struct AvailableDownloadJobRawInternal {
   pub creator_set_visibility: EntityVisibility,
 
   pub download_type: GenericDownloadType,
-  pub download_url: Option<String>,
+  pub download_url: String,
   pub title: String,
 
   pub status: JobStatus,

crates/lib/database_queries/src/queries/mod.rs

@@ -6,5 +6,6 @@ pub mod model_categories;
 pub mod tts;
 pub mod twitch;
 pub mod users;
+pub mod vocoder;
 pub mod voice_clone_requests;
 pub mod w2l;

crates/lib/database_queries/src/queries/vocoder/insert_vocoder_model.rs

@@ -0,0 +1,84 @@
+use std::path::Path;
+use anyhow::anyhow;
+use sqlx::MySqlPool;
+use container_common::anyhow_result::AnyhowResult;
+use reusable_types::entity_visibility::EntityVisibility;
+use reusable_types::vocoder_type::VocoderType;
+use crate::tokens::Tokens;
+
+pub struct Args<'a, P: AsRef<Path>> {
+  pub vocoder_type: VocoderType,
+
+  pub title: &'a str,
+
+  pub original_download_url: &'a str,
+  pub original_filename: &'a str,
+  pub file_size_bytes: u64,
+
+  pub creator_user_token: &'a str,
+  pub creator_ip_address: &'a str,
+  pub creator_set_visibility: EntityVisibility,
+
+  pub private_bucket_hash: &'a str,
+  pub private_bucket_object_name: P,
+
+  pub mysql_pool: &'a MySqlPool,
+}
+
+pub async fn insert_vocoder_model<P: AsRef<Path>>(
+  args: Args<'_, P>,
+) -> AnyhowResult<(u64, String)> {
+
+  let model_token = Tokens::new_vocoder_model()?;
+
+  let private_bucket_object_name = &private_bucket_object_name
+      .as_ref()
+      .display()
+      .to_string();
+
+  let query_result = sqlx::query!(
+        r#"
+INSERT INTO vocoder_models
+SET
+  token = ?,
+  vocoder_type = ?,
+  title = ?,
+  description_markdown = '',
+  description_rendered_html = '',
+  original_download_url = ?,
+  original_filename = ?,
+  file_size_bytes = ?,
+  creator_user_token = ?,
+  creator_ip_address_creation = ?,
+  creator_ip_address_last_update = ?,
+  creator_set_visibility = ?,
+  private_bucket_hash = ?,
+  private_bucket_object_name = ?,
+        "#,
+      &model_token,
+      args.vocoder_type,
+      &args.title,
+      &args.original_download_url,
+      &args.original_filename,
+      args.file_size_bytes,
+      &args.creator_user_token,
+      &args.creator_ip_address,
+      &args.creator_ip_address,
+      args.creator_set_visibility,
+      "todo",
+      "todo",
+    )
+      .execute(pool)
+      .await;
+
+  let record_id = match query_result {
+    Ok(res) => {
+      res.last_insert_id()
+    },
+    Err(err) => {
+      return Err(anyhow!("Mysql error: {:?}", err));
+    }
+  };
+
+  Ok((record_id, model_token))
+}

crates/lib/database_queries/src/queries/vocoder/mod.rs

@@ -0,0 +1 @@
+pub mod insert_vocoder_model;

crates/lib/database_queries/src/tokens.rs

@@ -13,6 +13,10 @@ impl Tokens {
     random_prefix_crockford_token("TM:", 15, false)
   }
 
+  pub fn new_vocoder_model() -> AnyhowResult<String> {
+    random_prefix_crockford_token("VM:", 15, false)
+  }
+
   pub fn new_tts_result() -> AnyhowResult<String> {
     random_prefix_crockford_token("TR:", 32, false)
   }

crates/lib/reusable_types/src/lib.rs

@@ -15,5 +15,6 @@
 
 pub mod entity_visibility;
 pub mod generic_download_type;
+pub mod vocoder_type;
 
 #[cfg(test)] pub mod test_helpers;

crates/lib/reusable_types/src/vocoder_type.rs

@@ -0,0 +1,17 @@
+#[derive(Clone, Copy, Eq, PartialEq, Debug, Deserialize, Serialize, sqlx::Type)]
+pub enum VocoderType {
+  #[serde(rename = "hifigan")]
+  #[sqlx(rename = "hifigan")]
+  HifiGan,
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::test_helpers::assert_serialization;
+  use crate::vocoder_type::VocoderType;
+
+  #[test]
+  fn test_serialization() {
+    assert_serialization(VocoderType::HifiGan, "hifigan");
+  }
+}

crates/lib/storage_buckets_common/src/bucket_path_unifier.rs

@@ -6,9 +6,10 @@ use std::path::PathBuf;
 /// different types of objects are stored.
 pub struct BucketPathUnifier {
   // TTS
-  pub tts_pretrained_vocoder_model_root: PathBuf,
   pub tts_synthesizer_model_root: PathBuf,
   pub tts_inference_output_root: PathBuf,
+  pub tts_pretrained_vocoder_model_root: PathBuf, // Pretrained, non-user uploaded vocoders
+  pub vocoder_model_root: PathBuf, // User-uploaded vocoders
   // W2L
   pub user_uploaded_w2l_templates_root: PathBuf,
   pub user_uploaded_audio_for_w2l_root: PathBuf,
@@ -19,20 +20,13 @@ pub struct BucketPathUnifier {
 
 impl BucketPathUnifier {
 
-  // TODO
-  //pub fn from_env_vars() -> AnyhowResult<Self> {
-  //  Ok(Self {
-  //    user_uploaded_audio_for_w2l_root: ,
-  //    user_uploaded_w2l_templates_root: ,
-  //  })
-  //}
-
   pub fn default_paths() -> Self {
     Self {
       // TTS
-      tts_pretrained_vocoder_model_root: PathBuf::from("/tts_pretrained_vocoders"),
       tts_synthesizer_model_root: PathBuf::from("/user_uploaded_tts_synthesizers"),
       tts_inference_output_root: PathBuf::from("/tts_inference_output"),
+      tts_pretrained_vocoder_model_root: PathBuf::from("/tts_pretrained_vocoders"), // Pretrained, non-user uploaded vocoders
+      vocoder_model_root: PathBuf::from("/user_uploaded_vocoders"),
       // W2L
       user_uploaded_audio_for_w2l_root: PathBuf::from("/user_uploaded_w2l_audio"),
       user_uploaded_w2l_templates_root: PathBuf::from("/user_uploaded_w2l_templates"),
@@ -44,12 +38,6 @@ impl BucketPathUnifier {
 
   // ==================== TTS MODELS (SYNTHESIZER + VOCODER) ==================== //
 
-  // TTS pretrained vocoder models.
-  // For now this will be limited, but once they're user-uploadable, we'll add more.
-  pub fn tts_pretrained_vocoders_path(&self, tts_vocoder_model_name: &str) -> PathBuf {
-    self.tts_pretrained_vocoder_model_root.join(tts_vocoder_model_name)
-  }
-
   pub fn tts_synthesizer_path(&self, tts_synthesizer_file_hash: &str) -> PathBuf {
     let hashed_path = Self::hashed_directory_path(tts_synthesizer_file_hash);
     let model_filename = format!("{}.pt", &tts_synthesizer_file_hash);
@@ -68,6 +56,25 @@ impl BucketPathUnifier {
         .join(model_filename)
   }
 
+  // ==================== VOCODER MODELS ==================== //
+
+  /// TTS pretrained vocoder models.
+  /// For now this will be limited, but once they're user-uploadable, we'll add more.
+  /// NB: These are vocoders that we uploaded without the FakeYou upload system!!
+  pub fn tts_pretrained_vocoders_path(&self, tts_vocoder_model_name: &str) -> PathBuf {
+    self.tts_pretrained_vocoder_model_root.join(tts_vocoder_model_name)
+  }
+
+  /// User-uploaded vocoders.
+  pub fn vocoder_path(&self, vocoder_file_hash: &str) -> PathBuf {
+    let hashed_path = Self::hashed_directory_path(vocoder_file_hash);
+    let model_filename = format!("{}.pt", &vocoder_file_hash);
+
+    self.vocoder_model_root
+        .join(hashed_path)
+        .join(model_filename)
+  }
+
   // ==================== TTS INFERENCE OUTPUT ==================== //
 
   /// This should include the string "vocodes" for downloaders.
@@ -179,9 +186,10 @@ mod tests {
   fn get_instance() -> BucketPathUnifier {
     BucketPathUnifier {
       // TTS
-      tts_pretrained_vocoder_model_root: PathBuf::from("/test_path_tts_vocoders"),
       tts_synthesizer_model_root: PathBuf::from("/test_path_synthesizers"),
       tts_inference_output_root: PathBuf::from("/test_path_tts_output"),
+      tts_pretrained_vocoder_model_root: PathBuf::from("/test_path_tts_vocoders"), // Pretrained, non-user uploaded vocoders
+      vocoder_model_root: PathBuf::from("/test_path_user_vocoders"), // User-uploaded vocoders
       // W2L
       user_uploaded_w2l_templates_root: PathBuf::from("/test_path_w2l_templates"),
       user_uploaded_audio_for_w2l_root: PathBuf::from("/test_path_w2l_audio"),
@@ -191,6 +199,7 @@ mod tests {
     }
   }
 
+  // NB: Pretrained, non-user uploaded vocoders
   #[test]
   fn test_tts_pretrained_vocoders_path() {
     let paths = get_instance();
@@ -198,6 +207,14 @@ mod tests {
       "/test_path_tts_vocoders/melgan.pth");
   }
 
+  // NB: User-uploaded vocoders
+  #[test]
+  fn test_vocoders_path() {
+    let paths = get_instance();
+    assert_eq!(paths.vocoder_path("foobar").to_str().unwrap(),
+      "/test_path_user_vocoders/f/o/o/foobar.pt");
+  }
+
   #[test]
   fn test_tts_synthesizer_path() {
     let paths = get_instance();

crates/service/generic_download_job/src/job_steps/process_hifigan_vocoder.rs

@@ -1,9 +1,17 @@
-use std::path::PathBuf;
-use log::info;
-use tempdir::TempDir;
+use anyhow::anyhow;
 use container_common::anyhow_result::AnyhowResult;
+use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
+use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
+use crate::{JobState, read_metadata_file};
 use database_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob;
-use crate::JobState;
+use log::{info, warn};
+use std::path::PathBuf;
+use tempdir::TempDir;
+use container_common::filesystem::check_file_exists::check_file_exists;
+use container_common::hashing::hash_file_sha2::hash_file_sha2;
+use database_queries::column_types::vocoder_type::VocoderType;
+use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::insert_tts_model;
+use database_queries::queries::vocoder::insert_vocoder_model::{Args, insert_vocoder_model};
 
 pub async fn process_hifigan_vocoder(
   job_state: &JobState,
@@ -21,32 +29,38 @@ pub async fn process_hifigan_vocoder(
 
   let output_metadata_fs_path = temp_dir.path().join("metadata.json");
 
-  match model_type {
-    TtsModelType::Tacotron2 => {
-      let result = job_state.tacotron_tts_check.execute(
-        &file_path,
-        &output_metadata_fs_path,
-        false,
-      );
+  // TODO TODO TODO CHECK MODEL VALIDITY
+  // TODO TODO TODO CHECK MODEL VALIDITY
+  // TODO TODO TODO CHECK MODEL VALIDITY
+  // TODO TODO TODO CHECK MODEL VALIDITY
+  // TODO TODO TODO CHECK MODEL VALIDITY
 
-      if let Err(e) = result {
-        safe_delete_temp_file(&file_path);
-        safe_delete_temp_directory(&temp_dir);
-      }
-    },
-    TtsModelType::Talknet => {
-      let result = job_state.talknet_tts_check.execute(
-        &file_path,
-        &output_metadata_fs_path,
-        false,
-      );
+  //match model_type {
+  //  TtsModelType::Tacotron2 => {
+  //    let result = job_state.tacotron_tts_check.execute(
+  //      &file_path,
+  //      &output_metadata_fs_path,
+  //      false,
+  //    );
 
-      if let Err(e) = result {
-        safe_delete_temp_file(&file_path);
-        safe_delete_temp_directory(&temp_dir);
-      }
-    },
-  }
+  //    if let Err(e) = result {
+  //      safe_delete_temp_file(&file_path);
+  //      safe_delete_temp_directory(&temp_dir);
+  //    }
+  //  },
+  //  TtsModelType::Talknet => {
+  //    let result = job_state.talknet_tts_check.execute(
+  //      &file_path,
+  //      &output_metadata_fs_path,
+  //      false,
+  //    );
+
+  //    if let Err(e) = result {
+  //      safe_delete_temp_file(&file_path);
+  //      safe_delete_temp_directory(&temp_dir);
+  //    }
+  //  },
+  //}
 
   // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
 
@@ -66,22 +80,19 @@ pub async fn process_hifigan_vocoder(
 
   // ==================== UPLOAD MODEL FILE ==================== //
 
-  info!("Uploading model to GCS...");
+  info!("Uploading HifiGan vocoder to GCS...");
 
   let private_bucket_hash = hash_file_sha2(&download_filename)?;
 
   info!("File hash: {}", private_bucket_hash);
 
-  let synthesizer_model_bucket_path = match model_type {
-    TtsModelType::Tacotron2 => job_state.bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash),
-    TtsModelType::Talknet => job_state.bucket_path_unifier.tts_zipped_synthesizer_path(&private_bucket_hash),
-  };
+  let model_bucket_path = job_state.bucket_path_unifier.vocoder_path(&private_bucket_hash);
 
-  info!("Destination bucket path: {:?}", &synthesizer_model_bucket_path);
+  info!("Destination bucket path: {:?}", &model_bucket_path);
 
   redis_logger.log_status("uploading model")?;
 
-  if let Err(e) = job_state.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await {
+  if let Err(e) = job_state.bucket_client.upload_filename(&model_bucket_path, &file_path).await {
     safe_delete_temp_file(&output_metadata_fs_path);
     safe_delete_temp_file(&file_path);
     safe_delete_temp_directory(&temp_dir);
@@ -98,13 +109,19 @@ pub async fn process_hifigan_vocoder(
   // ==================== SAVE RECORDS ==================== //
 
   info!("Saving model record...");
-  let (id, model_token) = insert_tts_model(
-    &job_state.mysql_pool,
-    job,
-    &private_bucket_hash,
-    synthesizer_model_bucket_path,
-    file_metadata.file_size_bytes)
-      .await?;
+  let (id, model_token) = insert_vocoder_model(Args {
+    vocoder_type: VocoderType::HifiGan, // TODO: This is the only type for now.
+    title: &job.title,
+    original_download_url: &job.download_url,
+    original_filename: "",
+    file_size_bytes: 0,
+    creator_user_token: "",
+    creator_ip_address: "",
+    creator_set_visibility: EntityVisibility::Public,
+    private_bucket_hash: "",
+    private_bucket_object_name: (),
+    mysql_pool: &job_state.mysql_pool
+  }).await?;
 
   job_state.badge_granter.maybe_grant_tts_model_uploads_badge(&job.creator_user_token)
       .await

crates/service/generic_download_job/src/job_steps/process_single_job.rs

@@ -12,6 +12,7 @@ use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use log::{info, warn};
 use std::path::PathBuf;
 use tempdir::TempDir;
+use config::is_bad_download_url::is_bad_download_url;
 use database_queries::queries::generic_download::job::mark_generic_download_job_done::mark_generic_download_job_done;
 use crate::job_steps::process_hifigan_vocoder::process_hifigan_vocoder;
 
@@ -41,16 +42,12 @@ pub async fn process_single_job(job_state: &JobState, job: &AvailableDownloadJob
 
   redis_logger.log_status("downloading model")?;
 
-  let download_url = job.download_url.as_ref()
-      .map(|c| c.to_string())
-      .unwrap_or("".to_string());
-
-  if is_bad_tts_model_download_url(&download_url)? {
-    warn!("Bad download URL: `{}`", &download_url);
-    return Err(anyhow!("Bad download URL: `{}`", &download_url));
+  if is_bad_download_url(&job.download_url)? {
+    warn!("Bad download URL: `{}`", &job.download_url);
+    return Err(anyhow!("Bad download URL: `{}`", &job.download_url));
   }
 
-  let download_filename = match job_state.google_drive_downloader.download_file(&download_url, &temp_dir).await {
+  let download_filename = match job_state.google_drive_downloader.download_file(&job.download_url, &temp_dir).await {
     Ok(filename) => filename,
     Err(e) => {
       safe_delete_temp_directory(&temp_dir);

crates/service/generic_download_job/src/script_execution/hifigan_model_check_command.rs

@@ -59,60 +59,17 @@ impl HifiGanModelCheckCommand {
       &command
     ];
 
-    if spawn_process {
-      // NB: This forks and returns immediately.
-      //let _child_pid = command_builder.spawn()?;
+    let mut p = Popen::create(&command_parts, PopenConfig {
+      //stdout: Redirection::Pipe,
+      //stderr: Redirection::Pipe,
+      ..Default::default()
+    })?;
 
-      let stdout_file = OpenOptions::new()
-          .read(true)
-          .write(true)
-          .create(true)
-          .truncate(true)
-          .open("/tmp/hifigan_upload_stdout.txt")?;
+    info!("Pid : {:?}", p.pid());
 
-      let stderr_file = OpenOptions::new()
-          .read(true)
-          .write(true)
-          .create(true)
-          .truncate(true)
-          .open("/tmp/hifigan_upload_stderr.txt")?;
+    let exit_status = p.wait()?;
 
-      let mut p = Popen::create(&command_parts, PopenConfig {
-        //stdout: Redirection::Pipe,
-        //stderr: Redirection::Pipe,
-        stdout: Redirection::File(stdout_file),
-        stderr: Redirection::File(stderr_file),
-        ..Default::default()
-      })?;
-
-      info!("Pid : {:?}", p.pid());
-
-      p.detach();
-
-    } else {
-      // NB: This is a blocking call.
-      /*let output = command_builder.output()?;
-
-      info!("Output status: {}", output.status);
-      info!("Stdout: {:?}", String::from_utf8(output.stdout));
-      error!("Stderr: {:?}", String::from_utf8(output.stderr));
-
-      if !output.status.success() {
-        bail!("Bad error code: {:?}", output.status);
-      }*/
-
-      let mut p = Popen::create(&command_parts, PopenConfig {
-        //stdout: Redirection::Pipe,
-        //stderr: Redirection::Pipe,
-        ..Default::default()
-      })?;
-
-      info!("Pid : {:?}", p.pid());
-
-      let exit_status = p.wait()?;
-
-      info!("Exit status: {:?}", exit_status);
-    }
+    info!("Exit status: {:?}", exit_status);
 
     Ok(())
   }