From bb0a4958efc6de855a9648046f2e7cf4c8baebe3 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 11 Oct 2023 18:19:40 -0400 Subject: [PATCH] Need to build and fix the container. --- .../voice_designer/voices/get_voice.rs | 9 ++- .../job/job_types/tts/vall_e_x/process_job.rs | 58 ++++++++++++------- .../job_types/tts/vall_e_x/validate_job.rs | 18 ++---- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs index 8895f73cb0..85bc40fa8f 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voices/get_voice.rs @@ -12,7 +12,8 @@ pub struct ZsVoice { pub title: String, pub ietf_language_tag: String, pub ietf_primary_language_subtag: String, - pub maybe_creator_user_token: Option, + pub bucket_hash: String, + pub maybe_creator_user_token: Option } pub async fn get_voice_by_token( @@ -68,7 +69,8 @@ pub async fn get_voice_by_token_with_connection( title: record.title, ietf_language_tag: record.ietf_language_tag, ietf_primary_language_subtag: record.ietf_primary_language_subtag, - maybe_creator_user_token: record.maybe_creator_user_token, + bucket_hash: record.bucket_hash, + maybe_creator_user_token: record.maybe_creator_user_token })) } @@ -85,6 +87,7 @@ async fn select_include_deleted( zv.ietf_language_tag, zv.ietf_primary_language_subtag, zv.maybe_creator_user_token, + zv.bucket_hash, zv.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility` FROM zs_voices as zv WHERE @@ -108,6 +111,7 @@ async fn select_without_deleted( zv.ietf_language_tag, zv.ietf_primary_language_subtag, zv.maybe_creator_user_token, + zv.bucket_hash, zv.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility` FROM zs_voices as zv WHERE @@ -126,5 +130,6 @@ pub struct RawVoice { ietf_language_tag: String, ietf_primary_language_subtag: String, maybe_creator_user_token: Option, + bucket_hash: String, creator_set_visibility: Visibility, } diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs index ff30c05beb..f870b50730 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -1,6 +1,7 @@ use anyhow::anyhow; use cloud_storage::bucket_client::BucketClient; -use log::{ error, info }; +use log::{ error, info, warn }; +use mysql_queries::queries::voice_designer::voices::get_voice::get_voice_by_token; use std::path::PathBuf; use std::time::Instant; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; @@ -11,15 +12,16 @@ use crate::job_dependencies::JobDependencies; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::InferenceArgs; use cloud_storage::bucket_path_unifier::BucketPathUnifier; - pub struct VoiceFile { pub filesystem_path: PathBuf, } -pub async fn download_voice_embedding_from_hash(object_path: str, - bucket_hash: str, - name: str, - private_bucket_client:BucketClient) -> Result { +pub async fn download_voice_embedding_from_hash( + object_path: str, + bucket_hash: str, + name: str, + private_bucket_client: BucketClient +) -> Result { let paths = BucketPathUnifier::default_paths(); let object_path = paths(bucket_hash, 0); let filesystem_path = format!("{}_weights.npz", name); @@ -43,6 +45,7 @@ pub async fn process_job( ) -> Result { let job = args.job; let deps = args.job_dependencies; + let mysql_pool = deps.mysql_pool; // get job args let text = match job.maybe_raw_inference_text { @@ -50,6 +53,24 @@ pub async fn process_job( None => { Err(ProcessSingleJobError::InvalidJob(anyhow!("Missing Text for Inference"))) } }; + // get voice token + + + // Get voice bucket hash - from voice token + let voice_lookup_result = get_voice_by_token(voice_token, false, &mysql_pool).await; + + let voice = match voice_lookup_result { + Ok(Some(voice)) => voice, + Ok(None) => { + warn!("Voice not found: {:?}", voice_token); + return Err(ProcessSingleJobError::Other(anyhow!(""))); + } + Err(err) => { + warn!("Error looking up voice: {:?}", err); + return Err(ProcessSingleJobError::Other(anyhow!(""))); + } + }; + // get some globals let job_progress_reporter = deps.job_progress_reporter .new_generic_inference(job.inference_job_token.as_str()) @@ -74,11 +95,19 @@ pub async fn process_job( } // Download embeddings file using embedding token - println!(""); - // Create a temp dir to download things to - // run inference + // Run Inference + let command_exit_status = + args.job_dependencies.job_type_details.vall_e_x.inference_command.execute_inference( + InferenceArgs { + input_embedding: String::from(""), // name of the embedding.npz in the tmp dir + input_text: String::from(""), // name of the text + /// --result_file: path to final file output + output_file_name: String::from(""), // output file name in the output folder + stderr_output_file: String::from(""), + } + ); // upload audio to bucket @@ -108,17 +137,6 @@ pub async fn process_job( let stderr_output_file = work_temp_dir.path().join("stderr.txt"); let inference_start_time = Instant::now(); - let command_exit_status = - args.job_dependencies.job_type_details.vall_e_x.inference_command.execute_inference( - InferenceArgs { - input_embedding: String::from(""), // name of the embedding.npz in the tmp dir - input_text: String::from(""), // name of the text - /// --result_file: path to final file output - output_file_name: String::from(""), // output file name in the output folder - stderr_output_file: String::from(""), - } - ); - // upload audio to public bucket ( for voice ) // return ok Err(ProcessSingleJobError::InvalidJob(anyhow!("this job flow is not yet implemented"))) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs index 8a7ab36aac..d18f22d577 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs @@ -1,15 +1,11 @@ use anyhow::anyhow; - use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::payloads::generic_inference_args::tts_payload::{TTSArgs}; - use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::job::job_types::lipsync::sad_talker::validate_job::JobArgs; -pub struct JobArgs<'a> { - text: String, - voice_token: String + +pub struct JobArgs { + voice_token: &String } pub fn validate_job(job: &AvailableInferenceJob) -> Result { @@ -46,12 +42,8 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result