diff --git a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json b/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json deleted file mode 100644 index fb10d9d281..0000000000 --- a/.sqlx/query-030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "MySQL", - "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 13 - }, - "nullable": [] - }, - "hash": "030e26fa7977555cdd04864d9b95c07196ab520cd8c63d30689b6021fa4b4ec1" -} diff --git a/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json new file mode 100644 index 0000000000..6a0e91bef8 --- /dev/null +++ b/.sqlx/query-8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n checksum_sha2 = ?,\n\n maybe_origin_filename = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 14 + }, + "nullable": [] + }, + "hash": "8f8b81d54aeabb31a200de98ba3106ac6bea906979c89355f805abb9650a7588" +} diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index 917a8622a0..558478772e 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -8,6 +8,7 @@ use errors::AnyhowResult; use crate::bucket_clients::get_bucket_clients; use crate::cli_args::parse_cli_args; use crate::seeding::model_weights::seed_weights; +use crate::seeding::seed_media_files::seed_media_files; use crate::seeding::tts_tacotron2::seed_tts_tacotron2; use crate::seeding::users::seed_user_accounts; use crate::seeding::voice_conversion::seed_voice_conversion; @@ -52,6 +53,7 @@ pub async fn main() -> AnyhowResult<()> { } seed_user_accounts(&pool).await?; + seed_media_files(&pool, maybe_bucket_clients.as_ref()).await?; seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?; seed_voice_conversion(&pool).await?; diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev_database_seed/src/seeding/mod.rs index cb462d17c9..c1100805b6 100644 --- a/crates/cli/dev_database_seed/src/seeding/mod.rs +++ b/crates/cli/dev_database_seed/src/seeding/mod.rs @@ -1,5 +1,6 @@ +pub mod model_weights; +pub mod seed_media_files; pub mod tts_tacotron2; pub mod users; pub mod voice_conversion; -pub mod zero_shot_tts; -pub mod model_weights; \ No newline at end of file +pub mod zero_shot_tts; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs new file mode 100644 index 0000000000..5bca8981e2 --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs @@ -0,0 +1,210 @@ +use std::path::Path; + +use log::{info, warn}; +use sqlx::{MySql, Pool}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use filesys::file_read_bytes::file_read_bytes; +use filesys::file_size::file_size; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs}; +use storyteller_root::get_seed_tool_data_root; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; + +pub async fn seed_media_files(mysql_pool: &Pool, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> { + info!("Seeding media files..."); + + // NB: This is idempotent and will only install the media files once. + // Delete the records if you want to re-seed. + let media_files = [ + // Images + (HANASHI_USER_TOKEN, "m_cn5t6ywm9k1ge9qzzxst5krc5tm08m", MediaFileType::Image, "media/images/face-animator-inputs/bill.jpg"), + (HANASHI_USER_TOKEN, "m_nf9xebrr8hd1bxkfy7src5k8nv2dzy", MediaFileType::Image, "media/images/face-animator-inputs/elon.jpg"), + (HANASHI_USER_TOKEN, "m_t17zh2r3ahs485b8m7c9q502cs6jqh", MediaFileType::Image, "media/images/face-animator-inputs/ernest.jpg"), + (HANASHI_USER_TOKEN, "m_vms09sby8tyw742y2s66a4w8f0w278", MediaFileType::Image, "media/images/face-animator-inputs/link.jpg"), + (HANASHI_USER_TOKEN, "m_s1z9krfzrscy92jgq3sxhpx1n4vxvc", MediaFileType::Image, "media/images/face-animator-inputs/ripley.jpg"), + (HANASHI_USER_TOKEN, "m_we7mnzbd76g4yn19dm6m53er0pjvew", MediaFileType::Image, "media/images/face-animator-inputs/san.png"), + (HANASHI_USER_TOKEN, "m_wahwz92nxxfg0x42dw9fdrhwq30yk0", MediaFileType::Image, "media/images/face-animator-inputs/zelda.jpg"), + // Audio + (HANASHI_USER_TOKEN, "m_21jxt406aattq2cna1xtgf5m4mjyyr", MediaFileType::Audio, "media/audio/voice-samples/biden.wav"), + (HANASHI_USER_TOKEN, "m_cxzabpxf88h10rshqraksxbqmhs41h", MediaFileType::Audio, "media/audio/voice-samples/brandon1.wav"), + (HANASHI_USER_TOKEN, "m_sq2vhrrmmbr079afk21543aa2jg53g", MediaFileType::Audio, "media/audio/voice-samples/brandon2.wav"), + (HANASHI_USER_TOKEN, "m_s61a8g3jnddr3xdg3gctsjxxk9jwja", MediaFileType::Audio, "media/audio/voice-samples/elon.wav"), + (HANASHI_USER_TOKEN, "m_e59fh75g76pvebx4qe4r4aymd93q30", MediaFileType::Audio, "media/audio/voice-samples/goku.wav"), + (HANASHI_USER_TOKEN, "m_ttq0ym8ht4v0z87zqz27fhadp0743c", MediaFileType::Audio, "media/audio/voice-samples/hillary.wav"), + (HANASHI_USER_TOKEN, "m_300bv6f141hv553fjf287p5gb76k37", MediaFileType::Audio, "media/audio/voice-samples/mario_impersonator.wav"), + (HANASHI_USER_TOKEN, "m_rd1ey2jem8a37r3vna2h9mht6zevs5", MediaFileType::Audio, "media/audio/voice-samples/rick.wav"), + (HANASHI_USER_TOKEN, "m_8n5epqfvs2vh60khap0s3ezem4epkc", MediaFileType::Audio, "media/audio/voice-samples/trump.wav"), + (HANASHI_USER_TOKEN, "m_g6dpwcpq51e6kvmpbbkdvgxfan9k1k", MediaFileType::Audio, "media/audio/voice-samples/trump_tts.wav"), + ]; + + let seed_tool_data_root = get_seed_tool_data_root(); + + for (user_token, media_file_token, media_file_type, subdirectory_path) in media_files { + let user_token = UserToken::new_from_str(user_token); + let media_file_token = MediaFileToken::new_from_str(media_file_token); + + let mut media_file_path = seed_tool_data_root.clone(); + media_file_path.push(subdirectory_path); + + let result = seed_model( + &mysql_pool, + &media_file_token, + &user_token, + media_file_type, + &media_file_path, + maybe_bucket_clients, + ).await; + + match result { + Ok(_) => info!("Seeded {:?}", media_file_path), + Err(err) => warn!(r#" + Could not seed media file {} , {:?} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the media file on a previous invocation!) + "#, media_file_token, subdirectory_path, err), + } + } + + Ok(()) +} + +async fn seed_model( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, + user_token: &UserToken, + media_file_type: MediaFileType, + media_file_path: &Path, + maybe_bucket_clients: Option<&BucketClients>, +) -> AnyhowResult<()> { + info!("Seeding Media file {:?} ...", media_file_path); + + let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + let mut maybe_bucket_prefix = None; + let mut maybe_bucket_extension = None; + + if media_file_exists(mysql_pool, media_file_token).await? { + info!("Media file already seeded: {:?}", media_file_token); + return Ok(()) + } + + if let Some(bucket_clients) = maybe_bucket_clients { + let bucket_details = seed_file_to_bucket(media_file_path, bucket_clients).await?; + bucket_hash = bucket_details.bucket_hash; + maybe_bucket_prefix = bucket_details.maybe_bucket_prefix; + maybe_bucket_extension = bucket_details.maybe_bucket_extension; + } + + let maybe_mime_type = get_mimetype_for_file(media_file_path)?; + let file_size_bytes = file_size(media_file_path)?; + let sha256_checksum = sha256_hash_file(media_file_path)?; + + let maybe_filename = media_file_path + .file_name() + .map(|name| name.to_str()) + .flatten(); + + insert_media_file_from_cli_tool(InsertArgs { + pool: &mysql_pool, + media_file_type, + maybe_mime_type, + file_size_bytes, + sha256_checksum: &sha256_checksum, + maybe_origin_filename: maybe_filename, + maybe_creator_user_token: Some(user_token), + creator_set_visibility: Visibility::Public, + public_bucket_directory_hash: &bucket_hash, + maybe_public_bucket_prefix: maybe_bucket_prefix.as_deref(), + maybe_public_bucket_extension: maybe_bucket_extension.as_deref(), + }).await?; + + Ok(()) +} + +async fn media_file_exists( + mysql_pool: &Pool, + media_file_token: &MediaFileToken, +) -> AnyhowResult { + + const CAN_SEE_DELETED : bool = true; + + let maybe_file = get_media_file( + &media_file_token, + CAN_SEE_DELETED, + mysql_pool + ).await?; + + Ok(maybe_file.is_some()) +} + +struct BucketDetails { + bucket_hash: String, + maybe_bucket_prefix: Option, + maybe_bucket_extension: Option, +} + +async fn seed_file_to_bucket( + media_file_path: &Path, + bucket_clients: &BucketClients, +) -> AnyhowResult { + + info!("Uploading media file {:?} ...", media_file_path); + + let maybe_bucket_prefix = Some("fakeyou_"); + + let maybe_bucket_extension = media_file_path + .extension() + .map(|extension| extension.to_str()) + .flatten(); + + let bucket_location = MediaFileBucketPath::generate_new(maybe_bucket_prefix, maybe_bucket_extension); + + let bucket_path = path_to_string(bucket_location.to_full_object_pathbuf()); + + info!("Reading media file: {:?}", media_file_path); + + let bytes = file_read_bytes(media_file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); + + info!("Uploading media file to bucket path: {:?}", bucket_path); + + let _r = bucket_clients.public.upload_file_with_content_type( + &bucket_path, + bytes.as_ref(), + mimetype + ).await?; + + Ok(BucketDetails { + bucket_hash: bucket_location.get_object_hash().to_string(), + maybe_bucket_prefix: maybe_bucket_prefix.map(|s| s.to_string()), + maybe_bucket_extension: maybe_bucket_extension.map(|s| s.to_string()), + }) +} + +// Temporary "test" to generate random tokens. +// #[cfg(test)] +// mod tests { +// use tokens::tokens::media_files::MediaFileToken; +// +// #[test] +// fn test() { +// // Print some randomly sampled tokens +// for _i in 0..50 { +// let token = MediaFileToken::generate(); +// println!("Token: {}", token); +// } +// // Deliberately fail the test to get the output +// assert_eq!(1, 2); +// } +// } diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs index 7a72bacf56..694780ffb9 100644 --- a/crates/cli/dev_upload_media_file/src/main.rs +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -123,6 +123,7 @@ pub async fn main() -> AnyhowResult<()> { maybe_mime_type, file_size_bytes: 0, // TODO sha256_checksum: "", // TODO + maybe_origin_filename: None, maybe_creator_user_token: None, creator_set_visibility, public_bucket_directory_hash: public_upload_path.get_object_hash(), diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs index 43a543db7d..fb180eba35 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -19,7 +19,9 @@ pub struct InsertArgs<'a> { pub file_size_bytes: u64, pub sha256_checksum: &'a str, - pub maybe_creator_user_token: Option, + pub maybe_origin_filename: Option<&'a str>, + + pub maybe_creator_user_token: Option<&'a UserToken>, pub creator_set_visibility: Visibility, pub public_bucket_directory_hash: &'a str, @@ -58,6 +60,8 @@ SET file_size_bytes = ?, checksum_sha2 = ?, + maybe_origin_filename = ?, + public_bucket_directory_hash = ?, maybe_public_bucket_prefix = ?, maybe_public_bucket_extension = ?, @@ -77,6 +81,8 @@ SET args.file_size_bytes, args.sha256_checksum, + args.maybe_origin_filename, + args.public_bucket_directory_hash, args.maybe_public_bucket_prefix, args.maybe_public_bucket_extension,