Seed media files

.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "MySQL",
-  "query": "\nINSERT INTO media_files\nSET\n  token = ?,\n\n  origin_category = ?,\n  origin_product_category = ?,\n\n  media_type = ?,\n  maybe_mime_type = ?,\n  file_size_bytes = ?,\n  checksum_sha2 = ?,\n\n  public_bucket_directory_hash = ?,\n  maybe_public_bucket_prefix = ?,\n  maybe_public_bucket_extension = ?,\n\n  maybe_creator_user_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?\n        ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 13
-    },
-    "nullable": []
-  },
-  "hash": "030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1"
-}

.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "\nINSERT INTO media_files\nSET\n  token = ?,\n\n  origin_category = ?,\n  origin_product_category = ?,\n\n  media_type = ?,\n  maybe_mime_type = ?,\n  file_size_bytes = ?,\n  checksum_sha2 = ?,\n\n  maybe_origin_filename = ?,\n\n  public_bucket_directory_hash = ?,\n  maybe_public_bucket_prefix = ?,\n  maybe_public_bucket_extension = ?,\n\n  maybe_creator_user_token = ?,\n  creator_ip_address = ?,\n\n  creator_set_visibility = ?\n        ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 14
+    },
+    "nullable": []
+  },
+  "hash": "8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588"
+}

crates/cli/dev_database_seed/src/main.rs

@@ -8,6 +8,7 @@ use errors::AnyhowResult;
 use crate::bucket_clients::get_bucket_clients;
 use crate::cli_args::parse_cli_args;
 use crate::seeding::model_weights::seed_weights;
+use crate::seeding::seed_media_files::seed_media_files;
 use crate::seeding::tts_tacotron2::seed_tts_tacotron2;
 use crate::seeding::users::seed_user_accounts;
 use crate::seeding::voice_conversion::seed_voice_conversion;
@@ -52,6 +53,7 @@ pub async fn main() -> AnyhowResult<()> {
   }
 
   seed_user_accounts(&pool).await?;
+  seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?;
   seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?;
   seed_voice_conversion(&pool).await?;
 

crates/cli/dev_database_seed/src/seeding/mod.rs

@@ -1,5 +1,6 @@
+pub mod model_weights;
+pub mod seed_media_files;
 pub mod tts_tacotron2;
 pub mod users;
 pub mod voice_conversion;
-pub mod zero_shot_tts;
-pub mod model_weights;
+pub mod zero_shot_tts;

crates/cli/dev_database_seed/src/seeding/seed_media_files.rs

@@ -0,0 +1,210 @@
+use std::path::Path;
+
+use log::{info, warn};
+use sqlx::{MySql, Pool};
+
+use buckets::public::media_files::original_file::MediaFileBucketPath;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::common::visibility::Visibility;
+use errors::AnyhowResult;
+use filesys::file_read_bytes::file_read_bytes;
+use filesys::file_size::file_size;
+use filesys::path_to_string::path_to_string;
+use hashing::sha256::sha256_hash_file::sha256_hash_file;
+use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
+use mimetypes::mimetype_for_file::get_mimetype_for_file;
+use mysql_queries::queries::media_files::get_media_file::get_media_file;
+use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs};
+use storyteller_root::get_seed_tool_data_root;
+use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::users::UserToken;
+
+use crate::bucket_clients::BucketClients;
+use crate::seeding::users::HANASHI_USER_TOKEN;
+
+pub async fn seed_media_files(mysql_pool: &Pool<MySql>, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> {
+  info!("Seeding media files...");
+
+  // NB: This is idempotent and will only install the media files once.
+  // Delete the records if you want to re-seed.
+  let media_files = [
+    // Images
+    (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/images/face-animator-inputs/bill.jpg"),
+    (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/images/face-animator-inputs/elon.jpg"),
+    (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/images/face-animator-inputs/ernest.jpg"),
+    (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/images/face-animator-inputs/link.jpg"),
+    (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/images/face-animator-inputs/ripley.jpg"),
+    (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/images/face-animator-inputs/san.png"),
+    (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/images/face-animator-inputs/zelda.jpg"),
+    // Audio
+    (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"),
+    (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"),
+    (HANASHI_USER_TOKEN, "m_sq2vhrrmmbr079afk21543aa2jg53g", MediaFileType::Audio, "media/audio/voice-samples/brandon2.wav"),
+    (HANASHI_USER_TOKEN, "m_s61a8g3jnddr3xdg3gctsjxxk9jwja", MediaFileType::Audio, "media/audio/voice-samples/elon.wav"),
+    (HANASHI_USER_TOKEN, "m_e59fh75g76pvebx4qe4r4aymd93q30", MediaFileType::Audio, "media/audio/voice-samples/goku.wav"),
+    (HANASHI_USER_TOKEN, "m_ttq0ym8ht4v0z87zqz27fhadp0743c", MediaFileType::Audio, "media/audio/voice-samples/hillary.wav"),
+    (HANASHI_USER_TOKEN, "m_300bv6f141hv553fjf287p5gb76k37", MediaFileType::Audio, "media/audio/voice-samples/mario_impersonator.wav"),
+    (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"),
+    (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"),
+    (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"),
+  ];
+
+  let seed_tool_data_root = get_seed_tool_data_root();
+
+  for (user_token, media_file_token, media_file_type, subdirectory_path) in media_files {
+    let user_token = UserToken::new_from_str(user_token);
+    let media_file_token = MediaFileToken::new_from_str(media_file_token);
+
+    let mut media_file_path = seed_tool_data_root.clone();
+    media_file_path.push(subdirectory_path);
+
+    let result = seed_model(
+      &mysql_pool,
+      &media_file_token,
+      &user_token,
+      media_file_type,
+      &media_file_path,
+      maybe_bucket_clients,
+    ).await;
+
+    match result {
+      Ok(_) => info!("Seeded {:?}", media_file_path),
+      Err(err) => warn!(r#"
+        Could not seed media file {} , {:?} : {:?}
+        (No worries: if there's a duplicate key error, we probably already
+        seeded the media file on a previous invocation!)
+      "#, media_file_token, subdirectory_path, err),
+    }
+  }
+
+  Ok(())
+}
+
+async fn seed_model(
+  mysql_pool: &Pool<MySql>,
+  media_file_token: &MediaFileToken,
+  user_token: &UserToken,
+  media_file_type: MediaFileType,
+  media_file_path: &Path,
+  maybe_bucket_clients: Option<&BucketClients>,
+) -> AnyhowResult<()> {
+  info!("Seeding Media file {:?} ...", media_file_path);
+
+  let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
+  let mut maybe_bucket_prefix = None;
+  let mut maybe_bucket_extension = None;
+
+  if media_file_exists(mysql_pool, media_file_token).await? {
+    info!("Media file already seeded: {:?}", media_file_token);
+    return Ok(())
+  }
+
+  if let Some(bucket_clients) = maybe_bucket_clients {
+    let bucket_details = seed_file_to_bucket(media_file_path, bucket_clients).await?;
+    bucket_hash = bucket_details.bucket_hash;
+    maybe_bucket_prefix = bucket_details.maybe_bucket_prefix;
+    maybe_bucket_extension = bucket_details.maybe_bucket_extension;
+  }
+
+  let maybe_mime_type = get_mimetype_for_file(media_file_path)?;
+  let file_size_bytes = file_size(media_file_path)?;
+  let sha256_checksum = sha256_hash_file(media_file_path)?;
+
+  let maybe_filename = media_file_path
+      .file_name()
+      .map(|name| name.to_str())
+      .flatten();
+
+  insert_media_file_from_cli_tool(InsertArgs {
+    pool: &mysql_pool,
+    media_file_type,
+    maybe_mime_type,
+    file_size_bytes,
+    sha256_checksum: &sha256_checksum,
+    maybe_origin_filename: maybe_filename,
+    maybe_creator_user_token: Some(user_token),
+    creator_set_visibility: Visibility::Public,
+    public_bucket_directory_hash: &bucket_hash,
+    maybe_public_bucket_prefix: maybe_bucket_prefix.as_deref(),
+    maybe_public_bucket_extension: maybe_bucket_extension.as_deref(),
+  }).await?;
+
+  Ok(())
+}
+
+async fn media_file_exists(
+  mysql_pool: &Pool<MySql>,
+  media_file_token: &MediaFileToken,
+) -> AnyhowResult<bool> {
+
+  const CAN_SEE_DELETED : bool = true;
+
+  let maybe_file = get_media_file(
+    &media_file_token,
+    CAN_SEE_DELETED,
+    mysql_pool
+  ).await?;
+
+  Ok(maybe_file.is_some())
+}
+
+struct BucketDetails {
+  bucket_hash: String,
+  maybe_bucket_prefix: Option<String>,
+  maybe_bucket_extension: Option<String>,
+}
+
+async fn seed_file_to_bucket(
+  media_file_path: &Path,
+  bucket_clients: &BucketClients,
+) -> AnyhowResult<BucketDetails> {
+
+  info!("Uploading media file {:?} ...", media_file_path);
+
+  let maybe_bucket_prefix = Some("fakeyou_");
+
+  let maybe_bucket_extension = media_file_path
+      .extension()
+      .map(|extension| extension.to_str())
+      .flatten();
+
+  let bucket_location = MediaFileBucketPath::generate_new(maybe_bucket_prefix, maybe_bucket_extension);
+
+  let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf());
+
+  info!("Reading media file: {:?}", media_file_path);
+
+  let bytes = file_read_bytes(media_file_path)?;
+  let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream");
+
+  info!("Uploading media file to bucket path: {:?}", bucket_path);
+
+  let _r = bucket_clients.public.upload_file_with_content_type(
+    &bucket_path,
+    bytes.as_ref(),
+    mimetype
+  ).await?;
+
+  Ok(BucketDetails {
+    bucket_hash: bucket_location.get_object_hash().to_string(),
+    maybe_bucket_prefix: maybe_bucket_prefix.map(|s| s.to_string()),
+    maybe_bucket_extension: maybe_bucket_extension.map(|s| s.to_string()),
+  })
+}
+
+// Temporary "test" to generate random tokens.
+// #[cfg(test)]
+// mod tests {
+//   use tokens::tokens::media_files::MediaFileToken;
+//
+//   #[test]
+//   fn test() {
+//     // Print some randomly sampled tokens
+//     for _i in 0..50 {
+//       let token = MediaFileToken::generate();
+//       println!("Token: {}", token);
+//     }
+//     // Deliberately fail the test to get the output
+//     assert_eq!(1, 2);
+//   }
+// }

crates/cli/dev_upload_media_file/src/main.rs

@@ -123,6 +123,7 @@ pub async fn main() -> AnyhowResult<()> {
     maybe_mime_type,
     file_size_bytes: 0, // TODO
     sha256_checksum: "", // TODO
+    maybe_origin_filename: None,
     maybe_creator_user_token: None,
     creator_set_visibility,
     public_bucket_directory_hash: public_upload_path.get_object_hash(),

crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs

@@ -19,7 +19,9 @@ pub struct InsertArgs<'a> {
   pub file_size_bytes: u64,
   pub sha256_checksum: &'a str,
 
-  pub maybe_creator_user_token: Option<UserToken>,
+  pub maybe_origin_filename: Option<&'a str>,
+
+  pub maybe_creator_user_token: Option<&'a UserToken>,
   pub creator_set_visibility: Visibility,
 
   pub public_bucket_directory_hash: &'a str,
@@ -58,6 +60,8 @@ SET
   file_size_bytes = ?,
   checksum_sha2 = ?,
 
+  maybe_origin_filename = ?,
+
   public_bucket_directory_hash = ?,
   maybe_public_bucket_prefix = ?,
   maybe_public_bucket_extension = ?,
@@ -77,6 +81,8 @@ SET
       args.file_size_bytes,
       args.sha256_checksum,
 
+      args.maybe_origin_filename,
+
       args.public_bucket_directory_hash,
       args.maybe_public_bucket_prefix,
       args.maybe_public_bucket_extension,