make idempotent

_sql/actions/truncate.sql

@@ -2,10 +2,13 @@
 -- noinspection SqlNoDataSourceInspectionForFile
 -- noinspection SqlResolveForFile
 
+-- DO NOT DO THIS IN PRODUCTION.
+-- Just placing this here to be easy to copy/paste in development.
 truncate zs_voices;
 truncate zs_voice_datasets;
 truncate zs_voice_dataset_samples;
 truncate media_files;
 truncate media_uploads;
 truncate generic_inference_jobs;
+truncate tts_models;
 

crates/cli/dev_database_seed/src/main.rs

@@ -9,6 +9,8 @@ use crate::bucket_clients::get_bucket_clients;
 use crate::cli_args::parse_cli_args;
 use crate::seeding::tts_tacotron2::seed_tts_tacotron2;
 use crate::seeding::users::seed_user_accounts;
+use crate::seeding::voice_conversion::seed_voice_conversion;
+use crate::seeding::zero_shot_tts::seed_zero_shot_tts;
 
 pub mod bucket_clients;
 pub mod cli_args;
@@ -48,8 +50,8 @@ pub async fn main() -> AnyhowResult<()> {
   }
 
   seed_user_accounts(&pool).await?;
-  //seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?;
-  //seed_voice_conversion(&pool).await?;
+  seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?;
+  seed_voice_conversion(&pool).await?;
   seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?;
 
   info!("Done!");

crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs

@@ -11,8 +11,10 @@ use filesys::file_read_bytes::file_read_bytes;
 use filesys::path_to_string::path_to_string;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
+use mysql_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token;
 use mysql_queries::queries::tts::tts_models::insert_tts_model_from_download_job::{insert_tts_model_from_download_job, InsertTtsModelFromDownloadJobArgs};
 use storyteller_root::get_seed_tool_data_root;
+use tokens::tokens::tts_models::TtsModelToken;
 use tokens::tokens::users::UserToken;
 
 use crate::bucket_clients::BucketClients;
@@ -23,7 +25,10 @@ pub async fn seed_tts_tacotron2(mysql_pool: &Pool<MySql>, maybe_bucket_clients:
 
   // NB: This is idempotent and will only install the models once.
   let tts_models = [
-    ("Son Goku V2 (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "models/tts/tacotron2/SonGokuV2.pth"),
+    ("Mai Valentine from Yugioh (by Vegito)", HANASHI_USER_TOKEN, "TM:cknxtj6mx3t3", "models/tts/tacotron2/MaiYugioh.pth"),
+    ("Nagito (by Vegito)", HANASHI_USER_TOKEN, "TM:tmhxapghwaf3", "models/tts/tacotron2/Nagito.pth"),
+    ("Son Goku V2 (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "TM:kbvaa4726jdw", "models/tts/tacotron2/SonGokuV2.pth"),
+    ("Vegeta (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "TM:kvv9xqashp6n", "models/tts/tacotron2/VegetaNew.pth"),
   ];
 
   let seed_tool_data_root = get_seed_tool_data_root();
@@ -31,14 +36,16 @@ pub async fn seed_tts_tacotron2(mysql_pool: &Pool<MySql>, maybe_bucket_clients:
   // TODO(bt,2023-11-27): DO NOT USE BucketPathUnifier for any new code. Only for TT2 and super legacy systems.
   let bucket_path_unifier = BucketPathUnifier::default_paths();
 
-  for (model_title, user_token, subdirectory_path) in tts_models {
+  for (model_title, user_token, tts_model_token, subdirectory_path) in tts_models {
     let user_token = UserToken::new_from_str(user_token);
+    let tts_model_token = TtsModelToken::new_from_str(tts_model_token);
 
     let mut model_file_path = seed_tool_data_root.clone();
     model_file_path.push(subdirectory_path);
 
     let result = seed_model(
       &mysql_pool,
+      &tts_model_token,
       model_title,
       &user_token,
       &model_file_path,
@@ -48,7 +55,11 @@ pub async fn seed_tts_tacotron2(mysql_pool: &Pool<MySql>, maybe_bucket_clients:
 
     match result {
       Ok(_) => info!("Seeded {}", model_title),
-      Err(err) => warn!("Could not seed TTS model {} : {:?}", model_title, err),
+      Err(err) => warn!(r#"
+        Could not TTS model {} : {:?}
+        (No worries: if there's a duplicate key error, we probably already
+        seeded the TTS model on a previous invocation!)
+      "#, model_title, err),
     }
   }
 
@@ -57,6 +68,7 @@ pub async fn seed_tts_tacotron2(mysql_pool: &Pool<MySql>, maybe_bucket_clients:
 
 async fn seed_model(
   mysql_pool: &Pool<MySql>,
+  tts_model_token: &TtsModelToken,
   model_title: &str,
   user_token: &UserToken,
   model_file_path: &Path,
@@ -68,30 +80,57 @@ async fn seed_model(
   let mut private_bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
   let mut private_bucket_object_name = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
 
+  if tts_model_exists(mysql_pool, tts_model_token).await? {
+    info!("Model already seeded: {:?}", tts_model_token);
+    return Ok(())
+  }
+
   if let Some(bucket_clients) = maybe_bucket_clients {
     let bucket_details = seed_file_to_bucket(model_file_path, bucket_clients, bucket_path_unifier).await?;
-
     private_bucket_hash = bucket_details.bucket_hash;
     private_bucket_object_name = bucket_details.bucket_object_name;
   }
 
+  let filename = model_file_path
+      .file_name()
+      .map(|name| name.to_str())
+      .flatten()
+      .unwrap_or("");
+
   insert_tts_model_from_download_job(InsertTtsModelFromDownloadJobArgs {
     tts_model_type: TtsModelType::Tacotron2,
     title: model_title,
     original_download_url: "https://example.com",
-    original_filename: "original_filename.zip",
+    original_filename: filename,
     file_size_bytes: 0,
     creator_user_token: user_token,
     creator_ip_address: "127.0.0.1",
     creator_set_visibility: Visibility::Public,
     private_bucket_hash: &private_bucket_hash,
     private_bucket_object_name: &private_bucket_object_name,
+    maybe_model_token: Some(&tts_model_token),
     mysql_pool,
-  }).await.map_err(|err| anyhow!("err: {:?}", err))?;
+  }).await?;
 
   Ok(())
 }
 
+async fn tts_model_exists(
+  mysql_pool: &Pool<MySql>,
+  tts_model_token: &TtsModelToken,
+) -> AnyhowResult<bool> {
+
+  const CAN_SEE_DELETED : bool = true;
+
+  let maybe_model = get_tts_model_by_token(
+    tts_model_token.as_str(),
+    CAN_SEE_DELETED,
+    mysql_pool
+  ).await?;
+
+  Ok(maybe_model.is_some())
+}
+
 struct BucketDetails {
   bucket_hash: String,
   bucket_object_name: String,
@@ -105,13 +144,15 @@ async fn seed_file_to_bucket(
 
   info!("Uploading Tacotron2 model file {:?} ...", model_file_path);
 
-  // TODO(bt,2023-11-27): This method of uploading model files is super deprecated and results in inconsistent pathing.
+  // TODO(bt,2023-11-27): DO NOT DO THIS OR COPY THIS PATTERN
+  //  This method of uploading model files is super deprecated and results in inconsistent pathing.
   //  Try to standardize on something resembling media_files going forward.
   let private_bucket_hash = sha256_hash_file(&model_file_path)?;
 
   info!("File hash: {}", private_bucket_hash);
 
-  // FIXME(bt,2023-11-27): 'bucket_path_unifier' is super deprecated. Do not use it anymore or for anything except TT2!
+  // FIXME(bt,2023-11-27): DO NOT COPY THIS PATTERN!
+  //  'bucket_path_unifier' is super deprecated. Do not use it anymore or for anything except TT2!
   let model_bucket_path = bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash);
 
   //let public_upload_path = MediaFileBucketPath::generate_new(

crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs

@@ -22,9 +22,10 @@ use tokens::tokens::users::UserToken;
 use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken;
 use tokens::tokens::zs_voices::ZsVoiceToken;
 
+use crate::bucket_clients::BucketClients;
 use crate::seeding::users::HANASHI_USERNAME;
 
-pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, maybe_bucket_client: Option<&BucketClient>) -> AnyhowResult<()> {
+pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> {
   info!("Seeding zero shot TTS...");
 
   let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? {
@@ -46,7 +47,7 @@ pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, maybe_bucket_client: O
   ];
 
   for (voice_name, bucket_hash, wav_file, user_token) in records {
-    create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, maybe_bucket_client).await?;
+    create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, maybe_bucket_clients).await?;
   }
 
   Ok(())
@@ -58,7 +59,7 @@ async fn create_voice_records(
   wav_file: &str,
   creator_user_token: &UserToken,
   mysql_pool: &Pool<MySql>,
-  maybe_bucket_client: Option<&BucketClient>,
+  maybe_bucket_clients: Option<&BucketClients>,
 ) -> AnyhowResult<(ZsVoiceToken, ZsVoiceDatasetToken)> {
   info!("Creating voice records for voice {} ...", voice_name);
 
@@ -76,8 +77,8 @@ async fn create_voice_records(
 
   let public_upload_path;
 
-  if let Some(bucket_client) = maybe_bucket_client {
-    public_upload_path = seed_file_to_bucket(wav_file, bucket_client).await?;
+  if let Some(bucket_clients) = maybe_bucket_clients {
+    public_upload_path = seed_file_to_bucket(wav_file, &bucket_clients.public).await?;
   } else {
     public_upload_path = MediaFileBucketPath::from_object_hash("fake", None, None);
   }

crates/lib/mysql_queries/src/queries/tts/tts_models/insert_tts_model_from_download_job.rs

@@ -25,15 +25,23 @@ pub struct InsertTtsModelFromDownloadJobArgs<'a, P: AsRef<Path>> {
   pub private_bucket_hash: &'a str,
   pub private_bucket_object_name: P,
 
+  /// In production code, send this as `None`.
+  /// Only provide an external model token for db integration tests and db seeding tools.
+  /// This allows for knowing the model token a priori.
+  pub maybe_model_token: Option<&'a TtsModelToken>,
+
   pub mysql_pool: &'a MySqlPool,
 }
 
 
 pub async fn insert_tts_model_from_download_job<P: AsRef<Path>>(
   args: InsertTtsModelFromDownloadJobArgs<'_, P>,
-) -> AnyhowResult<(u64, String)> {
+) -> AnyhowResult<(u64, TtsModelToken)> {
 
-  let model_token = TtsModelToken::generate().to_string();
+  let model_token = match args.maybe_model_token {
+    None => TtsModelToken::generate(),
+    Some(model_token) => model_token.clone(),
+  };
 
   let private_bucket_object_name = &args.private_bucket_object_name
       .as_ref()

crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs

@@ -114,6 +114,7 @@ pub async fn process_tacotron_model<'a, 'b>(
     creator_set_visibility: Visibility::Public, // TODO: All models default to public at start
     private_bucket_hash: &private_bucket_hash,
     private_bucket_object_name: &model_bucket_path,
+    maybe_model_token: None, // NB: This parameter is for internal testing only
     mysql_pool: &job_state.mysql_pool,
   }).await?;
 
@@ -125,7 +126,7 @@ pub async fn process_tacotron_model<'a, 'b>(
       })?;
 
   Ok(JobResults {
-    entity_token: Some(model_token),
+    entity_token: Some(model_token.to_string()),
     entity_type: Some(TtsModelType::Tacotron2.to_string()), // NB: This may be different from `GenericDownloadType` in the future!
   })
 }

crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs

@@ -144,6 +144,7 @@ pub async fn process_vits_model<'a, 'b>(
     creator_set_visibility: Visibility::Public, // TODO: All models default to public at start
     private_bucket_hash: &private_bucket_hash,
     private_bucket_object_name: "", // TODO: This should go away.
+    maybe_model_token: None, // NB: This parameter is for internal testing only
     mysql_pool: &job_state.mysql_pool,
   }).await?;
 
@@ -155,7 +156,7 @@ pub async fn process_vits_model<'a, 'b>(
       })?;
 
   Ok(JobResults {
-    entity_token: Some(model_token),
+    entity_token: Some(model_token.to_string()),
     entity_type: Some(TtsModelType::Vits.to_string()), // NB: This may be different from `GenericDownloadType` in the future!
   })
 }