diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs index d37f9dd9ac..887348ef88 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs @@ -15,6 +15,7 @@ pub struct ZsVoice { pub ietf_language_tag: String, pub ietf_primary_language_subtag: String, pub bucket_hash: String, + pub model_version: u64, pub maybe_creator_user_token: Option, pub creator_set_visibility: Visibility, pub created_at: DateTime, @@ -75,6 +76,7 @@ pub async fn get_voice_by_token_with_connection( ietf_language_tag: record.ietf_language_tag, ietf_primary_language_subtag: record.ietf_primary_language_subtag, bucket_hash: record.bucket_hash, + model_version: record.model_version as u64, // FIXME: Lazily widening. maybe_creator_user_token: record.maybe_creator_user_token, creator_set_visibility: record.creator_set_visibility, created_at: record.created_at, @@ -96,6 +98,7 @@ async fn select_include_deleted( zv.ietf_primary_language_subtag, zv.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`, zv.bucket_hash, + zv.model_version, zv.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, zv.created_at, zv.updated_at @@ -122,6 +125,7 @@ async fn select_without_deleted( zv.ietf_primary_language_subtag, zv.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`, zv.bucket_hash, + zv.model_version, zv.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, zv.created_at, zv.updated_at @@ -143,6 +147,7 @@ pub struct RawVoice { ietf_primary_language_subtag: String, maybe_creator_user_token: Option, bucket_hash: String, + model_version: u32, creator_set_visibility: Visibility, created_at: DateTime, updated_at: DateTime, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index 033c30a680..20b5dbd724 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -6,6 +6,8 @@ use anyhow::anyhow; use log::{error, info, warn}; use buckets::public::media_files::original_file::MediaFileBucketPath; +use buckets::public::zs_voices::directory::{ModelCategory, ModelType}; +use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; @@ -13,7 +15,7 @@ use filesys::file_size::file_size; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mysql_queries::queries::media_files::insert_media_file_from_zero_shot_tts::insert_media_file_from_zero_shot; use mysql_queries::queries::media_files::insert_media_file_from_zero_shot_tts::InsertArgs; -use mysql_queries::queries::voice_designer::voices::get_voice::get_voice_by_token; +use mysql_queries::queries::voice_designer::voices::get_voice::{get_voice_by_token, ZsVoice}; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::job_success_result::ResultEntity; @@ -99,18 +101,18 @@ pub async fn process_inference_voice( .map_err(|e| ProcessSingleJobError::from_io_error(e))?; let workdir = work_temp_dir.path().to_path_buf(); + let filename = "weights.npz".to_string(); - let file_name = format!("{}_weights.npz", &voice.title); + let mut downloaded_weights_path = work_temp_dir.path().to_path_buf(); + downloaded_weights_path.push(&filename); - // USE THIS LATER SINCE it requires specific typing ... - let voiceFile = download_voice_embedding_from_hash( - &voice.bucket_hash, - &file_name, + let voice_file = download_voice_embedding( + &voice, &deps.private_bucket_client, - &workdir + &downloaded_weights_path ).await?; - println!("voicefile path! {}", voiceFile.filesystem_path.to_string_lossy()); + println!("voice file path! {}", voice_file.filesystem_path.to_string_lossy()); // Download embeddings file using embedding token // Create a temp dir to download things to @@ -129,7 +131,7 @@ pub async fn process_inference_voice( args.job_dependencies.job_type_details.vall_e_x.inference_command.execute_inference( InferenceArgs { input_embedding_path: &workdir, - input_embedding_name: file_name, + input_embedding_name: filename, input_text: String::from(text), // text output_file_name: output_file_name.clone(), // output file name in the output folder stderr_output_file: &stderr_output_file, @@ -271,3 +273,36 @@ pub async fn download_voice_embedding_from_hash( Ok(voice_file) } +pub async fn download_voice_embedding( + voice: &ZsVoice, + private_bucket_client: &BucketClient, + download_path: &PathBuf, +) -> Result { + + let embedding_bucket_location = ZeroShotVoiceEmbeddingBucketPath::from_object_hash( + &voice.bucket_hash, + ModelCategory::Tts, + ModelType::VallEx, + voice.model_version + ); + + info!("Downloading embedding from: {:?}", &embedding_bucket_location.to_full_object_pathbuf()); + info!("Downloading to filesystem location: {:?}", download_path); + + let result = private_bucket_client.download_file_to_disk( + embedding_bucket_location.to_full_object_pathbuf(), + &download_path + ).await; + + if let Err(err) = result { + error!("could not download embedding file: {:?}", err); + return Err(ProcessSingleJobError::from_anyhow_error(err)); + } + + let voice_file = VoiceFile { + filesystem_path: PathBuf::from(&download_path.clone()), + }; + + Ok(voice_file) +} + diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 6ad6b9e63d..e80f975507 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -230,8 +230,23 @@ pub async fn get_inference_job_status_handler( .to_string() } InferenceCategory::TextToSpeech => { - // NB: TTS results receive the legacy treatment where their table only reports the full bucket path - result_details.public_bucket_location_or_hash + match result_details.entity_type.as_str() { + "media_file" => { + // NB: We're migrating TTS to media_files. + // Zero shot TTS uses media files. + // Legacy TT2 uses old pathing. + MediaFileBucketPath::from_object_hash( + &result_details.public_bucket_location_or_hash, + result_details.maybe_media_file_public_bucket_prefix.as_deref(), + result_details.maybe_media_file_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string() + } + _ => { + // NB: TTS results receive the legacy treatment where their table only reports the full bucket path + result_details.public_bucket_location_or_hash + } + } } InferenceCategory::VoiceConversion => { match result_details.entity_type.as_str() { @@ -287,6 +302,7 @@ fn filter_model_name(name: Option) -> Option { name.replace("sadtalker", "faceanimator") .replace("sad-talker", "face-animator") .replace("sad_talker", "face_animator") + .replace("vall_e_x", "voice_designer") }) }