From 0b3ea3af2a33a4ca3057db3c93b6082fe3205968 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 2 Oct 2023 16:15:25 -0400 Subject: [PATCH 01/19] Working on enqueue_tts_request assuming the prompt is around. --- .../voice_designer/enqueue_tts_request.rs | 75 +++++++++++++++++-- .../service/web/storyteller_web/src/routes.rs | 2 +- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 50f575e1c1..b14a611c7c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -1,15 +1,80 @@ - +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] use std::sync::Arc; - use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::error::ResponseError; use actix_web::http::StatusCode; - use log::{info, warn}; +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use tokens::jobs::inference::InferenceJobToken; +use crate::server_state::ServerState; +#[derive(Deserialize)] +pub struct EnqueueTTSRequest { + uuid_idempotency_token: String, + text: String +} -pub async fn enqueue_tts_request() -> HttpResponse { +#[derive(Serialize)] +pub struct EnqueueTTSRequestSuccessResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} + +#[derive(Debug)] +pub enum EnqueueTTSRequestError { + BadInput(String), + NotAuthorized, + ServerError, + RateLimited, +} + +impl ResponseError for EnqueueTTSRequestError { + fn status_code(&self) -> StatusCode { + match *self { + EnqueueTTSRequestError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueTTSRequestError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueTTSRequestError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueTTSRequestError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EnqueueTTSRequestError::BadInput(reason) => reason.to_string(), + EnqueueTTSRequestError::NotAuthorized => "unauthorized".to_string(), + EnqueueTTSRequestError::ServerError => "server error".to_string(), + EnqueueTTSRequestError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +impl std::fmt::Display for EnqueueTTSRequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + + +pub async fn enqueue_tts_request(http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result + { // Implementation for enqueuing a TTS request - HttpResponse::Ok().json("TTS request enqueued successfully") + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueTTSRequestError::ServerError + })?; + + HttpResponse::Ok().json("TTS request enqueued successfully"); } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index dea31f1ead..23baaa5b2b 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -174,7 +174,7 @@ pub fn add_routes (app: App) -> App app = add_subscription_routes(app); /* /v1/subscriptions/... */ // TODO find a long term feature flag solution, since this code is likely deployed into production we don't want the route found. - let enable_voice_designer_route = false; + let enable_voice_designer_route = true; if (enable_voice_designer_route) { app = add_voice_designer_routes(app); /* /v1/voice_designer */ } From 40d0d0d844e030326fb26cb12945f935f5fa00fa Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 2 Oct 2023 20:17:36 -0400 Subject: [PATCH 02/19] Changing up enqueue_infer_tts_handler to allow embedding tokens. --- .../enqueue_infer_tts_handler.rs | 1 + .../voice_designer/enqueue_tts_request.rs | 37 +++++++++++++++---- .../service/web/storyteller_web/src/routes.rs | 2 +- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index 29753f6587..74597efcac 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -73,6 +73,7 @@ const MAX_TTS_LENGTH : usize = 10_000; pub struct InferTtsRequest { uuid_idempotency_token: String, tts_model_token: String, + embedding_token: Option, // for any voice cloning based system that uses embeddings or prompts inference_text: String, creator_set_visibility: Option, is_storyteller_demo: Option, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index b14a611c7c..853e998c75 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -23,6 +23,7 @@ pub struct EnqueueTTSRequest { pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, + pub embedding_token: str } #[derive(Debug)] @@ -61,20 +62,40 @@ impl std::fmt::Display for EnqueueTTSRequestError { } } - +// TODO Delete later. +// Not gonna go this route, going to leverage enqueue_infer_tts_handler.rs instead. pub async fn enqueue_tts_request(http_request: HttpRequest, request: web::Json, server_state: web::Data>) -> Result { // Implementation for enqueuing a TTS request + + // do something with user session check if the user should even be able to access the end point + + - let mut mysql_connection = server_state.mysql_pool - .acquire() - .await - .map_err(|err| { - warn!("MySql pool error: {:?}", err); - EnqueueTTSRequestError::ServerError - })?; + // let mut mysql_connection = server_state.mysql_pool + // .acquire() + // .await + // .map_err(|err| { + // warn!("MySql pool error: {:?}", err); + // EnqueueTTSRequestError::ServerError + // })?; + +// check for session later + +// grab any data from other sources from GCB + +// rate limiter + + +// remap the json into data for lookup in the DB and for other processes, from the request + +// create the inference args here + +// enqueue a zero shot tts request here... + +// create the job record here! explore the table HttpResponse::Ok().json("TTS request enqueued successfully"); } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 23baaa5b2b..794760c6c3 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -1160,7 +1160,7 @@ fn add_voice_designer_routes (app:App)-> App ) .service( web::scope("/inference") - .route("/enqueue_tts", web::post().to(enqueue_tts_request)) + .route("/enqueue_tts", web::post().to(enqueue_infer_tts_handler)) .route("/enqueue_vc", web::post().to(enqueue_vc_request)) ) .service( From 773b96fb35718bdef59401d1d8ae8b09f622d821 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 2 Oct 2023 21:45:40 -0400 Subject: [PATCH 03/19] Stubbing out work in the handler Adding VallEX as a inference model type Test pass --- .../inference_model_type.rs | 15 ++-- .../animation/enqueue_lipsync_animation.rs | 8 +- .../enqueue_infer_tts_handler.rs | 1 - .../voice_designer/enqueue_tts_request.rs | 83 +++++++++++++------ .../service/web/storyteller_web/src/routes.rs | 3 +- 5 files changed, 72 insertions(+), 38 deletions(-) diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 79a6979b36..2a939038d6 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -19,20 +19,18 @@ use strum::EnumIter; pub enum InferenceModelType { #[serde(rename = "rvc_v2")] RvcV2, - // NB: sad_talker does use user-supplied models, so there is no "model token" #[serde(rename = "sad_talker")] SadTalker, - #[serde(rename = "so_vits_svc")] SoVitsSvc, - // TODO: Does this need to be "legacy_tacotron2" ? #[serde(rename = "tacotron2")] Tacotron2, - #[serde(rename = "vits")] Vits, + #[serde(rename = "vall_e_x")] + VallEX, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -48,6 +46,7 @@ impl InferenceModelType { Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", Self::Vits => "vits", + Self::VallEX => "vall_e_x" } } @@ -58,6 +57,7 @@ impl InferenceModelType { "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vits" => Ok(Self::Vits), + "vall_e_x" => Ok(Self::VallEX), _ => Err(format!("invalid value: {:?}", value)), } } @@ -71,6 +71,7 @@ impl InferenceModelType { InferenceModelType::SoVitsSvc, InferenceModelType::Tacotron2, InferenceModelType::Vits, + InferenceModelType::VallEX ]) } } @@ -87,6 +88,7 @@ mod tests { assert_serialization(InferenceModelType::SoVitsSvc, "so_vits_svc"); assert_serialization(InferenceModelType::Tacotron2, "tacotron2"); assert_serialization(InferenceModelType::Vits, "vits"); + assert_serialization(InferenceModelType::VallEX, "vall_e_x"); } #[test] @@ -96,6 +98,7 @@ mod tests { assert_eq!(InferenceModelType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(InferenceModelType::Vits.to_str(), "vits"); + assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x"); } #[test] @@ -105,18 +108,20 @@ mod tests { assert_eq!(InferenceModelType::from_str("so_vits_svc").unwrap(), InferenceModelType::SoVitsSvc); assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2); assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); + assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX); } #[test] fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 5); + assert_eq!(variants.len(), 6); assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); + assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs index c78e87ac68..33d5ebb940 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs @@ -234,7 +234,7 @@ pub async fn enqueue_lipsync_animation_handler( // TODO(bt): CHECK DATABASE FOR TOKENS! - let audio_source = { + let audio_source: LipsyncAnimationAudioSource = { if let Some(ref token) = request.audio_source.maybe_media_file_token { LipsyncAnimationAudioSource::media_file_token(token.as_str()) } else if let Some(ref token) = request.audio_source.maybe_media_upload_token { @@ -248,7 +248,7 @@ pub async fn enqueue_lipsync_animation_handler( } }; - let image_source = { + let image_source: LipsyncAnimationImageSource = { if let Some(ref token) = request.image_source.maybe_media_file_token { LipsyncAnimationImageSource::media_file_token(token.as_str()) } else if let Some(ref token) = request.image_source.maybe_media_upload_token { @@ -262,7 +262,7 @@ pub async fn enqueue_lipsync_animation_handler( let maybe_user_preferred_visibility : Option = maybe_user_session .as_ref() - .map(|user_session| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc + .map(|user_session: &users_component::utils::user_session_extended::UserSessionExtended| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc let set_visibility = request.creator_set_visibility .or(maybe_user_preferred_visibility) @@ -341,7 +341,7 @@ pub async fn enqueue_lipsync_animation_handler( EnqueueLipsyncAnimationError::ServerError })?; - let response = EnqueueLipsyncAnimationSuccessResponse { + let response: EnqueueLipsyncAnimationSuccessResponse = EnqueueLipsyncAnimationSuccessResponse { success: true, inference_job_token: job_token, }; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index 74597efcac..29753f6587 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -73,7 +73,6 @@ const MAX_TTS_LENGTH : usize = 10_000; pub struct InferTtsRequest { uuid_idempotency_token: String, tts_model_token: String, - embedding_token: Option, // for any voice cloning based system that uses embeddings or prompts inference_text: String, creator_set_visibility: Option, is_storyteller_demo: Option, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 853e998c75..0099979eea 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -12,20 +12,34 @@ use enums::by_table::generic_inference_jobs::inference_model_type::InferenceMode use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use tokens::jobs::inference::InferenceJobToken; use crate::server_state::ServerState; + + + +// TODO MOVE into own file. +use std::fmt::Debug; +use serde::Deserialize; +use serde::Serialize; +use crate::prefixes::EntityType; +/// The primary key for embeddings for voice cloning inference jobs. +#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] +#[sqlx(transparent)] +pub struct EmbeddingToken(String); + + #[derive(Deserialize)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, - text: String + text: String, + embedding_token: EmbeddingToken } - #[derive(Serialize)] pub struct EnqueueTTSRequestSuccessResponse { pub success: bool, - pub inference_job_token: InferenceJobToken, - pub embedding_token: str + pub inference_job_token: InferenceJobToken } + #[derive(Debug)] pub enum EnqueueTTSRequestError { BadInput(String), @@ -62,34 +76,47 @@ impl std::fmt::Display for EnqueueTTSRequestError { } } -// TODO Delete later. -// Not gonna go this route, going to leverage enqueue_infer_tts_handler.rs instead. -pub async fn enqueue_tts_request(http_request: HttpRequest, + +// Implementation for enqueuing a TTS request +// Reference enqueue_infer_tts_handler.rs for checks: rate limiting / user sessions +// insert generic inference job.rs +// Need to convert it to generic inference job. +pub async fn enqueue_tts_request( + http_request: HttpRequest, request: web::Json, - server_state: web::Data>) -> Result - { - // Implementation for enqueuing a TTS request + server_state: web::Data>) -> Result { // do something with user session check if the user should even be able to access the end point - - - // let mut mysql_connection = server_state.mysql_pool - // .acquire() - // .await - // .map_err(|err| { - // warn!("MySql pool error: {:?}", err); - // EnqueueTTSRequestError::ServerError - // })?; - -// check for session later - -// grab any data from other sources from GCB - -// rate limiter + // GET MY SQL + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueTTSRequestError::ServerError + })?; -// remap the json into data for lookup in the DB and for other processes, from the request +// TODO: check for session + +// TODO: Give investors priority + +// TODO: Add Rate Limiter + +// TODO: Look up model info? + +// TODO(bt): CHECK DATABASE FOR TOKENS? I am guessing we need to ensure those tokens exist because the files may not uploaded or are availible? +// get input from the container spec and create a object that similar to llipsync_payload.rs + +// PACKAGE JSON into RUST Struct, any smaller components +// remap from the request +// pass tokens from the request and create a payload that will have the information. +// check malformed json + +// Get up IP address + +// package as larger component args // create the inference args here @@ -97,5 +124,7 @@ pub async fn enqueue_tts_request(http_request: HttpRequest, // create the job record here! explore the table - HttpResponse::Ok().json("TTS request enqueued successfully"); +// Error handling 101 rust result type returned like so. + //Ok(HttpResponse::Ok().json("TTS request enqueued successfully")) + Err(EnqueueTTSRequestError::ServerError) } \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 794760c6c3..b6a15f4159 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -75,6 +75,7 @@ use crate::http_server::endpoints::tts::delete_tts_model::delete_tts_model_handl use crate::http_server::endpoints::tts::delete_tts_result::delete_tts_inference_result_handler; use crate::http_server::endpoints::tts::edit_tts_model::edit_tts_model_handler; use crate::http_server::endpoints::tts::edit_tts_result::edit_tts_inference_result_handler; +use crate::http_server::endpoints::tts::enqueue_infer_tts_handler; use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::enqueue_infer_tts_handler::enqueue_infer_tts_handler; use crate::http_server::endpoints::tts::enqueue_upload_tts_model::upload_tts_model_handler; use crate::http_server::endpoints::tts::get_pending_tts_inference_job_count::get_pending_tts_inference_job_count_handler; @@ -1160,7 +1161,7 @@ fn add_voice_designer_routes (app:App)-> App ) .service( web::scope("/inference") - .route("/enqueue_tts", web::post().to(enqueue_infer_tts_handler)) + .route("/enqueue_tts", web::post().to(enqueue_tts_request)) .route("/enqueue_vc", web::post().to(enqueue_vc_request)) ) .service( From ec4a17292958095d100c97498ebbf8b72afb4f1b Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 2 Oct 2023 23:08:28 -0400 Subject: [PATCH 04/19] Adding generic inference args and creating the job soon --- .../generic_inference_args.rs | 6 ++- .../payloads/generic_inference_args/mod.rs | 3 +- .../voice_designer/enqueue_tts_request.rs | 41 +++++++++++++++++-- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index eb26c9865b..0dafa2e85f 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -2,6 +2,7 @@ use enums::by_table::generic_inference_jobs::inference_category::InferenceCatego use errors::AnyhowResult; use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs; +use crate::payloads::generic_inference_args::tts_payload::TTSArgs; /// Used to encode extra state for the `generic_inference_jobs` table. /// This should act somewhat like a serialized protobuf stored inside a record. @@ -52,10 +53,11 @@ pub enum PolymorphicInferenceArgs { La (LipsyncArgs), /// Text to speech. (Short name to save space when serializing.) - Tts { + Tts(TTSArgs) + // No arguments yet. // It might be best to just not include this when not used. - }, + , /// Voice conversion. (Short name to save space when serializing.) Vc { /// Argument for so-vits-svc diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs index 2e526cc04c..37980d38d1 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs @@ -1,2 +1,3 @@ pub mod generic_inference_args; -pub mod lipsync_payload; \ No newline at end of file +pub mod lipsync_payload; +pub mod tts_payload; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 0099979eea..77dd976a67 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -2,6 +2,9 @@ #![forbid(unused_mut)] #![forbid(unused_variables)] +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; + use std::sync::Arc; use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::error::ResponseError; @@ -12,14 +15,13 @@ use enums::by_table::generic_inference_jobs::inference_model_type::InferenceMode use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use tokens::jobs::inference::InferenceJobToken; use crate::server_state::ServerState; - - +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; // TODO MOVE into own file. use std::fmt::Debug; use serde::Deserialize; use serde::Serialize; -use crate::prefixes::EntityType; + /// The primary key for embeddings for voice cloning inference jobs. #[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] #[sqlx(transparent)] @@ -122,7 +124,38 @@ pub async fn enqueue_tts_request( // enqueue a zero shot tts request here... -// create the job record here! explore the table +// create the job record here! explore the table insert the inference args in here as json! keep it short + + +let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + inference_category: InferenceCategory::TextToSpeech, + maybe_model_type: Some(InferenceModelType::VallEX), // NB: Model is static during inference + maybe_model_token: None, // NB: Model is static during inference + maybe_input_source_token: None, // TODO: Introduce a second foreign key ? + maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? + maybe_raw_inference_text: None, // No text + maybe_inference_args: Some(GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), + args: Some(PolymorphicInferenceArgs::La(inference_args)), + }), + maybe_creator_user_token: maybe_user_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: set_visibility, + priority_level, + requires_keepalive: plan.lipsync_requires_frontend_keepalive(), + is_debug_request, + maybe_routing_tag: maybe_routing_tag.as_deref(), + mysql_pool: &server_state.mysql_pool, +}).await; + +let job_token = match query_result { + Ok((job_token, _id)) => job_token, + Err(err) => { + warn!("New generic inference job creation DB error: {:?}", err); + return Err(EnqueueLipsyncAnimationError::ServerError); + } +}; // Error handling 101 rust result type returned like so. //Ok(HttpResponse::Ok().json("TTS request enqueued successfully")) From f7c636cf586df3b7d3ac371513727dca65526652 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 2 Oct 2023 23:56:52 -0400 Subject: [PATCH 05/19] Added TTS payloads should be able to enter a entry after a migration then follow the event to the inference dispatch and fill out the code there. --- .../generic_inference_args.rs | 6 +-- .../generic_inference_args/tts_payload.rs | 18 +++++++++ .../voice_designer/enqueue_tts_request.rs | 37 ++++++++++++------- 3 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index 0dafa2e85f..2c4e54fce7 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -54,7 +54,6 @@ pub enum PolymorphicInferenceArgs { /// Text to speech. (Short name to save space when serializing.) Tts(TTSArgs) - // No arguments yet. // It might be best to just not include this when not used. , @@ -142,12 +141,13 @@ mod tests { assert!(json.len() < 1000); } + + // TODO FIX THIS TEST! MICHAEL #[test] fn typical_tts_args_serialize() { let args = GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), - args: Some(PolymorphicInferenceArgs::Tts { - }), + args: Some(PolymorphicInferenceArgs::Tts()), }; let json = serde_json::ser::to_string(&args).unwrap(); diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs new file mode 100644 index 0000000000..06892accda --- /dev/null +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs @@ -0,0 +1,18 @@ +#[derive(Clone, Debug, Serialize, Deserialize)] + +pub struct TTSArgs { + #[serde(rename = "t")] + pub text: String, + #[serde(rename = "e")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_voice_token: Option // varchar 32 +} + +#[cfg(test)] +mod tests { + use crate::payloads::generic_inference_args::tts_payload::TTSArgs; + #[test] + fn test_tts_result() { + + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 77dd976a67..8aa48a973f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -10,8 +10,12 @@ use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use log::{info, warn}; -use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use http_server_common::request::get_request_ip::get_request_ip; + +use tokens::users::user::UserToken; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use mysql_queries::payloads::generic_inference_args::tts_payload::TTSArgs; + use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use tokens::jobs::inference::InferenceJobToken; use crate::server_state::ServerState; @@ -22,17 +26,12 @@ use std::fmt::Debug; use serde::Deserialize; use serde::Serialize; -/// The primary key for embeddings for voice cloning inference jobs. -#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] -#[sqlx(transparent)] -pub struct EmbeddingToken(String); - #[derive(Deserialize)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, text: String, - embedding_token: EmbeddingToken + embedding_token: String } #[derive(Serialize)] @@ -88,6 +87,11 @@ pub async fn enqueue_tts_request( request: web::Json, server_state: web::Data>) -> Result { + let is_debug_request = true; + let mut maybe_user_token : Option = None; + let mut priority_level ; + let disable_rate_limiter = false; // NB: Careful! + // do something with user session check if the user should even be able to access the end point // GET MY SQL @@ -117,8 +121,13 @@ pub async fn enqueue_tts_request( // check malformed json // Get up IP address +let ip_address = get_request_ip(&http_request); // package as larger component args +let mut inference_args = TTSArgs { + text: "Hello!".to_string(), + maybe_voice_token: Some("Voice Token.".to_string()) +}; // create the inference args here @@ -137,15 +146,15 @@ let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { maybe_raw_inference_text: None, // No text maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), - args: Some(PolymorphicInferenceArgs::La(inference_args)), + args: Some(PolymorphicInferenceArgs::Tts(inference_args)), }), maybe_creator_user_token: maybe_user_token.as_ref(), creator_ip_address: &ip_address, - creator_set_visibility: set_visibility, + creator_set_visibility: enums::common::visibility::Visibility::Public, priority_level, - requires_keepalive: plan.lipsync_requires_frontend_keepalive(), + requires_keepalive: true, // is_debug_request, - maybe_routing_tag: maybe_routing_tag.as_deref(), + maybe_routing_tag: "".to_string(), // TODO fix later mysql_pool: &server_state.mysql_pool, }).await; @@ -153,11 +162,11 @@ let job_token = match query_result { Ok((job_token, _id)) => job_token, Err(err) => { warn!("New generic inference job creation DB error: {:?}", err); - return Err(EnqueueLipsyncAnimationError::ServerError); + return Err(EnqueueTTSRequestError::ServerError); } }; // Error handling 101 rust result type returned like so. - //Ok(HttpResponse::Ok().json("TTS request enqueued successfully")) - Err(EnqueueTTSRequestError::ServerError) + Ok(HttpResponse::Ok().json("TTS request enqueued successfully")) + //Err(EnqueueTTSRequestError::ServerError) } \ No newline at end of file From e0145f0e70a5a46f0bb45a09adcb0eeb82abb50a Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Tue, 3 Oct 2023 19:09:09 -0400 Subject: [PATCH 06/19] Test this endpoint ensure that the record is created and you can see it grabbed. --- .../voice_designer/enqueue_tts_request.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 8aa48a973f..6228b71aac 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -89,7 +89,7 @@ pub async fn enqueue_tts_request( let is_debug_request = true; let mut maybe_user_token : Option = None; - let mut priority_level ; + let priority_level = 0; let disable_rate_limiter = false; // NB: Careful! // do something with user session check if the user should even be able to access the end point @@ -130,12 +130,8 @@ let mut inference_args = TTSArgs { }; // create the inference args here - // enqueue a zero shot tts request here... - // create the job record here! explore the table insert the inference args in here as json! keep it short - - let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { uuid_idempotency_token: &request.uuid_idempotency_token, inference_category: InferenceCategory::TextToSpeech, @@ -154,7 +150,7 @@ let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { priority_level, requires_keepalive: true, // is_debug_request, - maybe_routing_tag: "".to_string(), // TODO fix later + maybe_routing_tag: Some("Tag"), // TODO fix later what is this tag for? mysql_pool: &server_state.mysql_pool, }).await; @@ -166,7 +162,20 @@ let job_token = match query_result { } }; +let response: EnqueueTTSRequestSuccessResponse = EnqueueTTSRequestSuccessResponse { + success: true, + inference_job_token: job_token, +}; + + +let body = serde_json::to_string(&response) +.map_err(|_e| EnqueueTTSRequestError::ServerError)?; + + // Error handling 101 rust result type returned like so. - Ok(HttpResponse::Ok().json("TTS request enqueued successfully")) + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) + //Err(EnqueueTTSRequestError::ServerError) } \ No newline at end of file From 5de40b2dab1284362b7c77e7f409b86e0d024aa4 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Wed, 4 Oct 2023 16:30:46 -0400 Subject: [PATCH 07/19] Fix a build issues the payload sends exploring the job creation and inference side of it. --- .../generic_inference_args/tts_payload.rs | 2 +- .../src/job/job_types/tts/mod.rs | 1 + .../job/inference_job/src/job_dependencies.rs | 8 +++++ crates/service/job/inference_job/src/main.rs | 6 +++- .../animation/enqueue_lipsync_animation.rs | 2 ++ .../voice_designer/enqueue_tts_request.rs | 31 +++++++++---------- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs index 06892accda..6a3aeff9b8 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs @@ -5,7 +5,7 @@ pub struct TTSArgs { pub text: String, #[serde(rename = "e")] #[serde(skip_serializing_if = "Option::is_none")] - pub maybe_voice_token: Option // varchar 32 + pub voice_token: String // varchar 32 } #[cfg(test)] diff --git a/crates/service/job/inference_job/src/job/job_types/tts/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/mod.rs index 80c6c0b2c9..729c17f0ed 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/mod.rs @@ -1,3 +1,4 @@ pub mod process_single_tts_job; pub mod tacotron2_v2_early_fakeyou; pub mod vits; +pub mod vallEX; diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index b08251152e..a7ceb4b90e 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -161,6 +161,7 @@ pub struct JobTypeDetails { pub rvc_v2: RvcV2Details, pub so_vits_svc: SoVitsSvcDetails, pub sad_talker: SadTalkerDetails, + pub vallEX: VallEXDetails //pub tacotron2_modern: ..., //pub softvc: ..., } @@ -196,3 +197,10 @@ pub struct SadTalkerDetails { pub inference_command: SadTalkerInferenceCommand, pub ffmpeg_watermark_command: FfmpegLogoWatermarkCommand, } + +// TODO: we will probably want a command type of some kind that implements +// some kind of interface that we can just pass strings to. +pub struct VallEXDetails { + pub inference_command: VallEXInferenceCommand, + pub create_embedding_command: VallEXCreateEmbeddingCommand +} \ No newline at end of file diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 8c8b94d2b6..0123051b98 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -57,7 +57,7 @@ use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceComma use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::SoVitsSvcInferenceCommand; -use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, PretrainedModels, RvcV2Details, SadTalkerDetails, SoVitsSvcDetails, Tacotron2VocodesDetails, VitsDetails}; +use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, PretrainedModels, RvcV2Details, SadTalkerDetails, SoVitsSvcDetails, Tacotron2VocodesDetails, VitsDetails, VallEXDetails}; use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand; use crate::util::scoped_execution::ScopedExecution; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; @@ -307,6 +307,10 @@ async fn main() -> AnyhowResult<()> { hifigan_vocoder_model_filename, hifigan_superres_vocoder_model_filename, }, + vallEX: VallEXDetails { + inference_command: "", + create_embedding_command: "" + }, vits: VitsDetails { inference_command: vits_inference_command()?, }, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs index 33d5ebb940..e44f91e373 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs @@ -332,6 +332,8 @@ pub async fn enqueue_lipsync_animation_handler( } }; + // If you are using this file as a reference for another generic endpoint feature. + // this feature will be sunset so don't worry about this server_state.firehose_publisher.enqueue_lipsync_animation( maybe_user_token.as_ref(), &job_token) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 6228b71aac..8dee73bfb6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use actix_web::{HttpRequest, HttpResponse, web}; use actix_web::error::ResponseError; use actix_web::http::StatusCode; -use log::{info, warn}; +use log::{warn}; use http_server_common::request::get_request_ip::get_request_ip; use tokens::users::user::UserToken; @@ -87,21 +87,22 @@ pub async fn enqueue_tts_request( request: web::Json, server_state: web::Data>) -> Result { + println!("Recieved payload"); let is_debug_request = true; - let mut maybe_user_token : Option = None; + let maybe_user_token : Option = Some(UserToken::new_from_str(&"place holder")); let priority_level = 0; - let disable_rate_limiter = false; // NB: Careful! + //let disable_rate_limiter = false; // NB: Careful! // do something with user session check if the user should even be able to access the end point // GET MY SQL - let mut mysql_connection = server_state.mysql_pool - .acquire() - .await - .map_err(|err| { - warn!("MySql pool error: {:?}", err); - EnqueueTTSRequestError::ServerError - })?; + // let mut mysql_connection = server_state.mysql_pool + // .acquire() + // .await + // .map_err(|err| { + // warn!("MySql pool error: {:?}", err); + // EnqueueTTSRequestError::ServerError + // })?; // TODO: check for session @@ -123,10 +124,10 @@ pub async fn enqueue_tts_request( // Get up IP address let ip_address = get_request_ip(&http_request); -// package as larger component args -let mut inference_args = TTSArgs { - text: "Hello!".to_string(), - maybe_voice_token: Some("Voice Token.".to_string()) +// package as larger component args should always have an embedding token .. +let inference_args = TTSArgs { + text: request.text, + maybe_voice_token: request.embedding_token }; // create the inference args here @@ -167,11 +168,9 @@ let response: EnqueueTTSRequestSuccessResponse = EnqueueTTSRequestSuccessRespons inference_job_token: job_token, }; - let body = serde_json::to_string(&response) .map_err(|_e| EnqueueTTSRequestError::ServerError)?; - // Error handling 101 rust result type returned like so. Ok(HttpResponse::Ok() .content_type("application/json") From 4f5f6cfe51fcffb7fba549f7493baf0bd9b7675e Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 5 Oct 2023 13:29:51 -0400 Subject: [PATCH 08/19] Getting the pieces ready for inference_job --- .../config/inference-job.development.env | 12 +++ .../src/job/job_types/tts/mod.rs | 2 +- .../src/job/job_types/tts/vall_e_x/mod.rs | 2 + .../tts/vall_e_x/model_downloaders.rs | 1 + .../vall_e_x/vall_e_x_inference_command.rs | 99 +++++++++++++++++++ .../job/inference_job/src/job_dependencies.rs | 6 +- crates/service/job/inference_job/src/main.rs | 5 + 7 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 554d092fde..05f1227d14 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -9,6 +9,18 @@ HTTP_BIND_ADDRESS=0.0.0.0:11223 # NB: Disable if not testing Redis REDIS_FOR_KEEPALIVE_URL="redis://localhost/0" +# ================================= # +# # +# Vall-E-X # +# # +# ================================= # + +# should rename my folder +# abs path to the folder that contains the inference code need to get that container in the repo need basepath? relative to everyones env +VALL_E_X_INFERENCE_ROOT_DIRECTORY = "/home/tensor/code/Storyteller/storyteller-ml/tts/VALL-E-X" +VALL_E_X_INFERENCE_COMMAND = "python3 main.py" +VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .env/bin/activate" + # ================================= # # # # RVC (v2) # diff --git a/crates/service/job/inference_job/src/job/job_types/tts/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/mod.rs index 729c17f0ed..26d581e703 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/mod.rs @@ -1,4 +1,4 @@ pub mod process_single_tts_job; pub mod tacotron2_v2_early_fakeyou; pub mod vits; -pub mod vallEX; +pub mod vall_e_x; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs new file mode 100644 index 0000000000..727c4b6166 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs @@ -0,0 +1,2 @@ +pub mod model_downloaders; +pub mod vall_e_x_inference_command; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs new file mode 100644 index 0000000000..ea843fdaec --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs @@ -0,0 +1 @@ +use crate::util::model_downloader::ModelDownloader; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs new file mode 100644 index 0000000000..0ae673cd39 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs @@ -0,0 +1,99 @@ +use std::collections::HashSet; +use std::env; +use std::ffi::OsString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::anyhow; +use log::info; +use once_cell::sync::Lazy; +use subprocess::{Popen, PopenConfig, Redirection}; + +use container_common::anyhow_result::AnyhowResult; +use filesys::path_to_string::path_to_string; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; + +use crate::job::job_loop::command_exit_status::CommandExitStatus; + +#[derive(Clone)] +pub enum ExecutableOrCommand { + /// Eg. `inference.py` + Executable(PathBuf), + + /// Eg. `python3 inference.py` + Command(String), +} + + +pub struct InferenceArgs<'s, P: AsRef> { + /// --driven_audio: path to the input audio + pub input_audio: P, + + /// --source_image: path to the input image (or video) + pub input_image: P, + + /// --result_dir: path to directory work is performed + pub work_dir: P, + + /// --result_file: path to final file output + pub output_file: P, + + pub stderr_output_file: P, + + /// --still: less animation + pub make_still: bool, + + /// --enhancer: "gfpgan" + pub maybe_enhancer: Option<&'s str>, + + /// --preprocess: "crop", etc. + pub maybe_preprocess: Option<&'s str>, + + // TODO: Other SadTalker args + } + + +pub impl VallEXInferenceCommand { + pub fn new>( + root_code_directory: P, + executable_or_command: ExecutableOrCommand, + maybe_virtual_env_activation_command: Option<&str>, + maybe_default_config_path: Option

, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + alternate_checkpoint_dir: Option + ) -> Self { + Self { + root_code_directory: root_code_directory.as_ref().to_path_buf(), + executable_or_command, + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + } + } +} + +pub impl VallEXCreateEmbeddingCommand { + pub fn new>( + root_code_directory: P, + executable_or_command: ExecutableOrCommand, + maybe_virtual_env_activation_command: Option<&str>, + maybe_default_config_path: Option

, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + alternate_checkpoint_dir: Option + ) -> Self { + Self { + root_code_directory: root_code_directory.as_ref().to_path_buf(), + executable_or_command, + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + } + } +} \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index a7ceb4b90e..340ed78c50 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -23,6 +23,10 @@ use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDown use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; + +use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXInferenceCommand; +use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; + use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::SoVitsSvcInferenceCommand; @@ -203,4 +207,4 @@ pub struct SadTalkerDetails { pub struct VallEXDetails { pub inference_command: VallEXInferenceCommand, pub create_embedding_command: VallEXCreateEmbeddingCommand -} \ No newline at end of file +} diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 0123051b98..d9af8260bb 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -53,10 +53,14 @@ use crate::job::job_loop::main_loop::main_loop; use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; + use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; + use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::SoVitsSvcInferenceCommand; + use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, PretrainedModels, RvcV2Details, SadTalkerDetails, SoVitsSvcDetails, Tacotron2VocodesDetails, VitsDetails, VallEXDetails}; use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand; use crate::util::scoped_execution::ScopedExecution; @@ -308,6 +312,7 @@ async fn main() -> AnyhowResult<()> { hifigan_superres_vocoder_model_filename, }, vallEX: VallEXDetails { + downloaders: VallEXDownloaders::build_all_from_env(), inference_command: "", create_embedding_command: "" }, From a2c196587f7c73607a27da30e080d61f13c96e2d Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 5 Oct 2023 22:29:25 -0400 Subject: [PATCH 09/19] Added Inference Command Stub Added Create Embedding Command Stub Need to implement job process: Download Embedding File aka voice token. model_downloaders is all good. validate_job. --- .../lipsync/sad_talker/process_job.rs | 4 +- .../job_types/tts/process_single_tts_job.rs | 9 +- .../tts/vall_e_x/download_embedding_file.rs | 0 .../tts/vall_e_x/model_downloaders.rs | 92 ++++++++++++++- .../job/job_types/tts/vall_e_x/process_job.rs | 73 ++++++++++++ .../job_types/tts/vall_e_x/validate_job.rs | 0 .../vall_e_x/vall_e_x_inference_command.rs | 109 ++++++++---------- .../job/inference_job/src/job_dependencies.rs | 6 +- crates/service/job/inference_job/src/main.rs | 9 +- 9 files changed, 233 insertions(+), 69 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/download_embedding_file.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 8830f96ae1..65abdfed72 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -77,7 +77,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result) -> Result return Err(ProcessSingleJobError::Other(anyhow!("tts model not found: {:?}", tts_model_token))), Some(model) => model, }; - let job_success_result = match tts_model.tts_model_type { + let job_success_result: JobSuccessResult = match tts_model.tts_model_type { TtsModelType::Tacotron2 => { tacotron2_v2_early_fakeyou::process_job::process_job(ProcessJobArgs { job_dependencies, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/download_embedding_file.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/download_embedding_file.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs index ea843fdaec..aaa6906e10 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs @@ -1 +1,91 @@ -use crate::util::model_downloader::ModelDownloader; \ No newline at end of file +use crate::util::model_downloader::ModelDownloader; + +pub struct VallEXDownloaders { + pub vall_e_x: String, + pub whisper_lg: String, + pub whisper_md: String, + pub vocos_encodec_24khz: String +} + +crate::impl_model_downloader!( + VallEXModelMapping, + "Vall-e-x Model Mapping", + "VALL_E_X_MAPPING_BUCKET_PATH", + "/dependencies/zero_shot_tts/vall-e-x_1.0", + "VALL_E_X_MAPPING_FILESYSTEM_PATH", + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0" + ); + + crate::impl_model_downloader!( + WhisperLargeModelMapping, + "whisper Large Model Mapping", + "WHISPER_LARGE_MAPPING_BUCKET_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/whisper-large", + "WHISPER_LARGE_MAPPING_FILESYSTEM_PATH", + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0/whisper-large" + ); + + crate::impl_model_downloader!( + WhisperMediumModelMapping, + "whisper Medium Model Mapping", + "WHISPER_MEDIUM_TENSOR_BUCKET_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/whisper-medium", + "WHISPER_MEDIUM_TENSOR_FILESYSTEM_PATH", + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0/whisper-medium" + ); + + crate::impl_model_downloader!( + VocosEncodecModelMapping, + "Vocos Encodec Model Mapping", + "VOCOS_ENCODEC_TENSOR_BUCKET_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin", + "VOCOS_ENCODEC_TENSOR_FILESYSTEM_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin" + ); + +impl VallEXDownloaders { + pub fn build_all_from_env() -> Self { + Self { + vall_e_x: VallEXModelMapping::from_env(), + whisper_lg: WhisperLargeModelMapping::from_env(), + whisper_md: WhisperMediumModelMapping::from_env(), + vocos_encodec_24khz: VocosEncodecModelMapping::from_env() + } + } + + pub fn all_downloaders(&self) -> Vec<&dyn ModelDownloader> { + vec![ + &self.vall_e_x, + &self.whisper_lg, + &self.whisper_md, + &self.vocos_encodec_24khz, + ] + } + } + + + + mod test { + // use std::path::Path; + + // use crate::job::job_types::lipsync::sad_talker::model_downloaders::{SadTalkerModelFirstMapping, SadTalkerModelSecondMapping}; + // use crate::util::model_downloader::ModelDownloader; + + // #[test] + // fn test_downloader_model_first_mapping() { + // let model = SadTalkerModelFirstMapping::default(); + // assert_eq!(model.get_model_name(), "sad talker model first mapping"); + // assert_eq!(model.get_cloud_bucket_path(), "/animation_sadtalker/model/mapping_00109-model.pth.tar"); + // assert_eq!(model.get_filesystem_path(), Path::new("/tmp/downloads/sadtalker/mapping_00109-model.pth.tar")); + // } + + // #[test] + // fn test_downloader_model_second_mapping() { + // let model = SadTalkerModelSecondMapping::default(); + // assert_eq!(model.get_model_name(), "sad talker model second mapping"); + // assert_eq!(model.get_cloud_bucket_path(), "/animation_sadtalker/model/mapping_00229-model.pth.tar"); + // assert_eq!(model.get_filesystem_path(), Path::new("/tmp/downloads/sadtalker/mapping_00229-model.pth.tar")); + // } + + } + \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs new file mode 100644 index 0000000000..2afc067c12 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -0,0 +1,73 @@ +use std::fs::read_to_string; +use std::time::Instant; + +use anyhow::anyhow; +use log::{error, info, warn}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use container_common::filesystem::check_file_exists::check_file_exists; +use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::file_size::file_size; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; +use tokens::users::user::UserToken; + +use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +//use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; +//use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; +//use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; + +use crate::job_dependencies::JobDependencies; + +pub struct VALL_E_XProcessJobArgs<'a> { + pub job_dependencies: &'a JobDependencies, + pub job: &'a AvailableInferenceJob, + } + + pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result { + + // get job args and dependencies and execute them with the inputs + let job = args.job; + let deps = args.job_dependencies; + + let mut job_progress_reporter = args.job_dependencies + .job_progress_reporter + .new_generic_inference(job.inference_job_token.as_str()) + .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; + + // validate the inputs on the job + let job_args = validate_job(job)?; + + + // Need to download the models + info!("Download models (if not present)..."); + let mut i : usize = 0; + for downloader in deps.job_type_details.sad_talker.downloaders.all_downloaders() { + + // Temporary debugging + info!("Downloader {}", i); + i = i + 1; + + let result = downloader.download_if_not_on_filesystem( + &args.job_dependencies.private_bucket_client, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + ).await; + + if let Err(e) = result { + error!("could not download: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + } + + // Download embeddings + + // + + + } \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs index 0ae673cd39..38e272a3e5 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs @@ -20,80 +20,73 @@ use crate::job::job_loop::command_exit_status::CommandExitStatus; pub enum ExecutableOrCommand { /// Eg. `inference.py` Executable(PathBuf), - /// Eg. `python3 inference.py` Command(String), } + pub struct InferenceArgs<'s, P: AsRef> { /// --driven_audio: path to the input audio - pub input_audio: P, + pub input_audio: P, // Change to files /// --source_image: path to the input image (or video) - pub input_image: P, - + pub input_embedding: P, + pub input_text: P, /// --result_dir: path to directory work is performed pub work_dir: P, - /// --result_file: path to final file output pub output_file: P, - pub stderr_output_file: P, - - /// --still: less animation - pub make_still: bool, - - /// --enhancer: "gfpgan" - pub maybe_enhancer: Option<&'s str>, - - /// --preprocess: "crop", etc. - pub maybe_preprocess: Option<&'s str>, - - // TODO: Other SadTalker args } +pub struct VallEXInferenceCommand { -pub impl VallEXInferenceCommand { - pub fn new>( - root_code_directory: P, - executable_or_command: ExecutableOrCommand, - maybe_virtual_env_activation_command: Option<&str>, - maybe_default_config_path: Option

, - maybe_docker_options: Option, - maybe_execution_timeout: Option, - alternate_checkpoint_dir: Option - ) -> Self { - Self { - root_code_directory: root_code_directory.as_ref().to_path_buf(), - executable_or_command, - maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), - maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), - maybe_docker_options, - maybe_execution_timeout, - alternate_checkpoint_dir, - } - } +} +// pub impl VallEXInferenceCommand { +// pub fn new>( +// root_code_directory: P, +// executable_or_command: ExecutableOrCommand, +// maybe_virtual_env_activation_command: Option<&str>, +// maybe_default_config_path: Option

, +// maybe_docker_options: Option, +// maybe_execution_timeout: Option, +// alternate_checkpoint_dir: Option +// ) -> Self { +// Self { +// root_code_directory: root_code_directory.as_ref().to_path_buf(), +// executable_or_command, +// maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), +// maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), +// maybe_docker_options, +// maybe_execution_timeout, +// alternate_checkpoint_dir, +// } +// } +// } + +pub struct VallEXCreateEmbeddingCommand { + } -pub impl VallEXCreateEmbeddingCommand { - pub fn new>( - root_code_directory: P, - executable_or_command: ExecutableOrCommand, - maybe_virtual_env_activation_command: Option<&str>, - maybe_default_config_path: Option

, - maybe_docker_options: Option, - maybe_execution_timeout: Option, - alternate_checkpoint_dir: Option - ) -> Self { - Self { - root_code_directory: root_code_directory.as_ref().to_path_buf(), - executable_or_command, - maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), - maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), - maybe_docker_options, - maybe_execution_timeout, - alternate_checkpoint_dir, - } - } -} \ No newline at end of file +// pub impl VallEXCreateEmbeddingCommand { +// pub fn new>( +// root_code_directory: P, +// executable_or_command: ExecutableOrCommand, +// maybe_virtual_env_activation_command: Option<&str>, +// maybe_default_config_path: Option

, +// maybe_docker_options: Option, +// maybe_execution_timeout: Option, +// alternate_checkpoint_dir: Option +// ) -> Self { +// Self { +// root_code_directory: root_code_directory.as_ref().to_path_buf(), +// executable_or_command, +// maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), +// maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), +// maybe_docker_options, +// maybe_execution_timeout, +// alternate_checkpoint_dir, +// } +// } +// } \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 340ed78c50..9f87447404 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -24,8 +24,8 @@ use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::Sa use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; -use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXInferenceCommand; -use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; @@ -165,7 +165,7 @@ pub struct JobTypeDetails { pub rvc_v2: RvcV2Details, pub so_vits_svc: SoVitsSvcDetails, pub sad_talker: SadTalkerDetails, - pub vallEX: VallEXDetails + pub vall_e_x: VallEXDetails //pub tacotron2_modern: ..., //pub softvc: ..., } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index d9af8260bb..f25760e7f0 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -55,7 +55,10 @@ use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::Sa use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; + use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; +use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders; use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; @@ -311,10 +314,10 @@ async fn main() -> AnyhowResult<()> { hifigan_vocoder_model_filename, hifigan_superres_vocoder_model_filename, }, - vallEX: VallEXDetails { + vall_e_x: VallEXDetails { downloaders: VallEXDownloaders::build_all_from_env(), - inference_command: "", - create_embedding_command: "" + inference_command: VallEXInferenceCommand, + create_embedding_command: VallEXCreateEmbeddingCommand, }, vits: VitsDetails { inference_command: vits_inference_command()?, From ba76acec3562beb16cd45fcc2bb8fad273758f10 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 5 Oct 2023 23:05:27 -0400 Subject: [PATCH 10/19] some more fixes to model downloaders --- .../job/job_types/tts/vall_e_x/model_downloaders.rs | 2 +- .../endpoints/voice_designer/enqueue_tts_request.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs index aaa6906e10..1504b64206 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs @@ -40,7 +40,7 @@ crate::impl_model_downloader!( "VOCOS_ENCODEC_TENSOR_BUCKET_PATH", "dependencies/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin", "VOCOS_ENCODEC_TENSOR_FILESYSTEM_PATH", - "dependencies/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin" + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin" ); impl VallEXDownloaders { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 8dee73bfb6..079ddb8028 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -89,12 +89,12 @@ pub async fn enqueue_tts_request( println!("Recieved payload"); let is_debug_request = true; - let maybe_user_token : Option = Some(UserToken::new_from_str(&"place holder")); + let maybe_user_token : Option = Some(UserToken::new_from_str(&"place holder")); // TODO fix this let priority_level = 0; //let disable_rate_limiter = false; // NB: Careful! // do something with user session check if the user should even be able to access the end point - + // CAN address this soon just getting everything out of the way to just enqueue the job. // GET MY SQL // let mut mysql_connection = server_state.mysql_pool // .acquire() @@ -104,6 +104,8 @@ pub async fn enqueue_tts_request( // EnqueueTTSRequestError::ServerError // })?; + // return errors on payload not being correct + //Err(EnqueueTTSRequestError::ServerError) # missing voice token // TODO: check for session @@ -127,7 +129,7 @@ let ip_address = get_request_ip(&http_request); // package as larger component args should always have an embedding token .. let inference_args = TTSArgs { text: request.text, - maybe_voice_token: request.embedding_token + voice_token: request.embedding_token }; // create the inference args here @@ -149,7 +151,7 @@ let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { creator_ip_address: &ip_address, creator_set_visibility: enums::common::visibility::Visibility::Public, priority_level, - requires_keepalive: true, // + requires_keepalive: true, // do we need this? I think so is_debug_request, maybe_routing_tag: Some("Tag"), // TODO fix later what is this tag for? mysql_pool: &server_state.mysql_pool, @@ -176,5 +178,5 @@ let body = serde_json::to_string(&response) .content_type("application/json") .body(body)) - //Err(EnqueueTTSRequestError::ServerError) + } \ No newline at end of file From 195a2eeb9cd0be85bf6705db08a8717b2c07ec5e Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 00:49:42 -0400 Subject: [PATCH 11/19] Outlined the things needed to be done in process_job --- .../job/job_types/tts/process_single_tts_job.rs | 14 +++++++++++++- .../src/job/job_types/tts/vall_e_x/mod.rs | 5 ++++- .../src/job/job_types/tts/vall_e_x/process_job.rs | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 4777ae556a..33f781b6e1 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -10,8 +10,11 @@ use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::{tacotron2_v2_early_fakeyou, vits}; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::process_job::ProcessJobArgs; use crate::job::job_types::tts::vits::process_job::VitsProcessJobArgs; +use crate::job::job_types::tts::vall_e_x::process_job::VALLEXProcessJobArgs; use crate::job_dependencies::JobDependencies; +use super::vall_e_x; + pub async fn process_single_tts_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { let tts_model_token = match job.maybe_model_token.as_deref() { @@ -60,7 +63,16 @@ pub async fn process_single_tts_job(job_dependencies: &JobDependencies, job: &Av // Look for the Zeroshot job type - + let job_success_result = match job.maybe_model_type { + Some(InferenceModelType::VallEX) => { + vall_e_x::process_job::process_job(VALLEXProcessJobArgs { + job_dependencies, + job, + }).await? + } + Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), + None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), + }; // let these run anyways ? diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs index 727c4b6166..b9f326ac0b 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs @@ -1,2 +1,5 @@ pub mod model_downloaders; -pub mod vall_e_x_inference_command; \ No newline at end of file +pub mod vall_e_x_inference_command; +pub mod process_job; +pub mod validate_job; +pub mod download_embedding_file; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs index 2afc067c12..e0e3d5c65e 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -25,7 +25,7 @@ use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job_dependencies::JobDependencies; -pub struct VALL_E_XProcessJobArgs<'a> { +pub struct VALLEXProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, } From 9e3a3b79f93336c755a12e793fdbaf9312dfe623 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 09:49:09 -0400 Subject: [PATCH 12/19] job process wip --- .../job/job_types/tts/vall_e_x/process_job.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs index e0e3d5c65e..f362621636 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -48,7 +48,7 @@ pub struct VALLEXProcessJobArgs<'a> { // Need to download the models info!("Download models (if not present)..."); let mut i : usize = 0; - for downloader in deps.job_type_details.sad_talker.downloaders.all_downloaders() { + for downloader in deps.job_type_details.vall_e_x.downloaders.all_downloaders() { // Temporary debugging info!("Downloader {}", i); @@ -65,9 +65,21 @@ pub struct VALLEXProcessJobArgs<'a> { } } - // Download embeddings + // Download embeddings files - // + // rinr inference + // ==================== TEMP DIR ==================== // + // create a temp dir + + // run inference + + // upload audio to bucket + + // deletetemp dir files + + // save result ? + + } \ No newline at end of file From c6eefc41ecf32d91c20989a83c6579d4aecaa2b4 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 10:55:51 -0400 Subject: [PATCH 13/19] Ok fixed the build now working on job running. --- .../src/payloads/generic_inference_args/tts_payload.rs | 1 - .../endpoints/voice_designer/enqueue_tts_request.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs index 6a3aeff9b8..482a72813c 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/tts_payload.rs @@ -4,7 +4,6 @@ pub struct TTSArgs { #[serde(rename = "t")] pub text: String, #[serde(rename = "e")] - #[serde(skip_serializing_if = "Option::is_none")] pub voice_token: String // varchar 32 } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index 079ddb8028..ecaefb3b42 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -128,8 +128,8 @@ let ip_address = get_request_ip(&http_request); // package as larger component args should always have an embedding token .. let inference_args = TTSArgs { - text: request.text, - voice_token: request.embedding_token + text: request.text.clone(), + voice_token: request.embedding_token.clone() }; // create the inference args here From dda05ba22cacaf12b7e075809d8ac42e33584c64 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 13:01:17 -0400 Subject: [PATCH 14/19] Have to pull master for the lastest seed changes --- .../src/job/job_types/tts/vall_e_x/process_job.rs | 8 +++++--- .../src/job/job_types/tts/vall_e_x/validate_job.rs | 9 +++++++++ .../endpoints/voice_designer/enqueue_tts_request.rs | 1 - 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs index f362621636..aa455d0e15 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -21,7 +21,8 @@ use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; //use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; //use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; -//use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; + +use crate::job::job_types::tts::vall_e_x::validate_job::validate_job; use crate::job_dependencies::JobDependencies; @@ -41,13 +42,14 @@ pub struct VALLEXProcessJobArgs<'a> { .new_generic_inference(job.inference_job_token.as_str()) .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; - // validate the inputs on the job - let job_args = validate_job(job)?; + // validate the inputs on the job shouldn't this happen at json + //let job_args = validate_job(job)?; // Need to download the models info!("Download models (if not present)..."); let mut i : usize = 0; + for downloader in deps.job_type_details.vall_e_x.downloaders.all_downloaders() { // Temporary debugging diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs index e69de29bb2..81177b61ae 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs @@ -0,0 +1,9 @@ +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; + +use crate::job::{job_types::lipsync::sad_talker::validate_job::JobArgs, job_loop::process_single_job_error::ProcessSingleJobError}; + + + +pub fn validate_job(job: &AvailableInferenceJob) -> Result { + +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs index ecaefb3b42..64de4a3359 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/enqueue_tts_request.rs @@ -26,7 +26,6 @@ use std::fmt::Debug; use serde::Deserialize; use serde::Serialize; - #[derive(Deserialize)] pub struct EnqueueTTSRequest { uuid_idempotency_token: String, From ae899d8aca1bc19b24fa365877ef5e68262f1362 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 15:21:38 -0400 Subject: [PATCH 15/19] Fixed most of the job and implemented it. Need to run the container this might be fun haha ... --- .../config/inference-job.development.env | 2 +- .../tts/vall_e_x/model_downloaders.rs | 8 +- .../job/job_types/tts/vall_e_x/process_job.rs | 13 +- .../vall_e_x/vall_e_x_inference_command.rs | 385 +++++++++++++++--- .../job/inference_job/src/job_dependencies.rs | 2 + crates/service/job/inference_job/src/main.rs | 4 +- 6 files changed, 352 insertions(+), 62 deletions(-) diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 05f1227d14..1b14ed66b9 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -20,7 +20,7 @@ REDIS_FOR_KEEPALIVE_URL="redis://localhost/0" VALL_E_X_INFERENCE_ROOT_DIRECTORY = "/home/tensor/code/Storyteller/storyteller-ml/tts/VALL-E-X" VALL_E_X_INFERENCE_COMMAND = "python3 main.py" VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .env/bin/activate" - +VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE = "" # fill this out for dev? # ================================= # # # # RVC (v2) # diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs index 1504b64206..28887ccce8 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs @@ -1,10 +1,10 @@ use crate::util::model_downloader::ModelDownloader; pub struct VallEXDownloaders { - pub vall_e_x: String, - pub whisper_lg: String, - pub whisper_md: String, - pub vocos_encodec_24khz: String + pub vall_e_x: VallEXModelMapping, + pub whisper_lg: WhisperLargeModelMapping, + pub whisper_md: WhisperMediumModelMapping, + pub vocos_encodec_24khz: VocosEncodecModelMapping } crate::impl_model_downloader!( diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs index aa455d0e15..3230a6efe2 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -26,12 +26,14 @@ use crate::job::job_types::tts::vall_e_x::validate_job::validate_job; use crate::job_dependencies::JobDependencies; + +// This will download everything get into the root host OS then ... will invoke inference using the pathes from the files invoked pub struct VALLEXProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, } - pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result { + pub async fn process_job(args: VALLEXProcessJobArgs<'_>) -> Result { // get job args and dependencies and execute them with the inputs let job = args.job; @@ -84,4 +86,13 @@ pub struct VALLEXProcessJobArgs<'a> { + } + + + #[cfg(test)] + mod test { + #[test] + fn values() { + print!("{}","Hello!"); + } } \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs index 38e272a3e5..7eb2f12737 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs @@ -24,69 +24,346 @@ pub enum ExecutableOrCommand { Command(String), } +// These environment vars are not copied over to the subprocess +// TODO/FIXME(bt, 2023-05-28): This is horrific security! +static IGNORED_ENVIRONMENT_VARS : Lazy> = Lazy::new(|| { + let env_var_names= [ + "MYSQL_URL", + "ACCESS_KEY", + "SECRET_KEY", + "NEWRELIC_API_KEY", + ]; + env_var_names.iter() + .map(|value| value.to_string()) + .collect::>() +}); -pub struct InferenceArgs<'s, P: AsRef> { +// The inference command assumes everything has been setup to run inference +// so we have all the files downloaded for the inputs. +pub struct InferenceArgs> { /// --driven_audio: path to the input audio - pub input_audio: P, // Change to files - - /// --source_image: path to the input image (or video) - pub input_embedding: P, - pub input_text: P, - /// --result_dir: path to directory work is performed - pub work_dir: P, + pub input_embedding: P, // name of the embedding.npz in the prompts folder? + pub input_text: P, // name of the text /// --result_file: path to final file output - pub output_file: P, - pub stderr_output_file: P, + pub output_file_name: P, // output file name in the output folder + pub stderr_output_file: P } +#[derive(Clone)] pub struct VallEXInferenceCommand { - + /// Where the code lives + root_code_directory: PathBuf, + /// A single executable script or a much larger bash command. + executable_or_command: ExecutableOrCommand, + /// eg. `source python/bin/activate` + maybe_virtual_env_activation_command: Option, + /// Optional default config file to use + maybe_default_config_path: Option, + /// If this is run under Docker (eg. in development), these are the options. + maybe_docker_options: Option, + /// If the execution should be ended after a certain point. + maybe_execution_timeout: Option, + /// Inference arg. + /// --checkpoint_dir: optional location for checkpoints directory + pub alternate_checkpoint_dir: Option } -// pub impl VallEXInferenceCommand { -// pub fn new>( -// root_code_directory: P, -// executable_or_command: ExecutableOrCommand, -// maybe_virtual_env_activation_command: Option<&str>, -// maybe_default_config_path: Option

, -// maybe_docker_options: Option, -// maybe_execution_timeout: Option, -// alternate_checkpoint_dir: Option -// ) -> Self { -// Self { -// root_code_directory: root_code_directory.as_ref().to_path_buf(), -// executable_or_command, -// maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), -// maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), -// maybe_docker_options, -// maybe_execution_timeout, -// alternate_checkpoint_dir, -// } -// } -// } +impl VallEXInferenceCommand { + pub fn new>( + root_code_directory: P, + executable_or_command: ExecutableOrCommand, + maybe_virtual_env_activation_command: Option<&str>, + maybe_default_config_path: Option

, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + alternate_checkpoint_dir: Option + ) -> Self { + Self { + root_code_directory: root_code_directory.as_ref().to_path_buf(), + executable_or_command, + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + } + } + + pub fn from_env() -> AnyhowResult { + let root_code_directory = easyenv::get_env_pathbuf_required( + "VALL_E_X_INFERENCE_ROOT_DIRECTORY")?; + + let maybe_inference_command = easyenv::get_env_string_optional( + "VALL_E_X_INFERENCE_COMMAND"); + + + // EVENTUALLY should remove the check + // Optional, eg. `./infer.py`. Typically we'll use the command form instead. + let maybe_inference_executable = easyenv::get_env_pathbuf_optional( + "VALL_E_X_INFERENCE_EXECUTABLE"); + + let executable_or_command = match maybe_inference_command { + Some(command) => ExecutableOrCommand::Command(command), + None => match maybe_inference_executable { + Some(executable) => ExecutableOrCommand::Executable(executable), + None => return Err(anyhow!("neither command nor executable passed")), + }, + }; + + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( + "VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND"); + + let maybe_default_config_path = easyenv::get_env_pathbuf_optional( + "VALL_E_X_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH"); + + let maybe_execution_timeout = + easyenv::get_env_duration_seconds_optional("TIMEOUT_SECONDS"); + + + // Probably for local + let maybe_docker_options = easyenv::get_env_string_optional( + "VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_environment_variables: None, + maybe_gpu: Some(DockerGpu::All), + } + }); + + // Override for --checkpoint_dir at inference time + let alternate_checkpoint_dir = easyenv::get_env_pathbuf_optional( + "SAD_TALKER_ALTERNATE_CHECKPOINT_PATH"); + + Ok(Self { + root_code_directory, + executable_or_command, + maybe_virtual_env_activation_command, + maybe_default_config_path, + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + }) + } + + pub fn execute_inference>( + &self, + args: InferenceArgs

, + ) -> CommandExitStatus { + match self.do_execute_inference(args) { + Ok(exit_status) => exit_status, + Err(error) => CommandExitStatus::FailureWithReason { reason: format!("error: {:?}", error) }, + } + } + + fn do_execute_inference>( + &self, + args: InferenceArgs

, + ) -> AnyhowResult { + + let mut command = String::new(); + command.push_str(&format!("cd {}", path_to_string(&self.root_code_directory))); + + if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() { + command.push_str(" && "); + command.push_str(venv_command); + command.push_str(" "); + } + + command.push_str(" && "); + + match self.executable_or_command { + ExecutableOrCommand::Executable(ref executable) => { + command.push_str(&path_to_string(executable)); + command.push_str(" "); + } + ExecutableOrCommand::Command(ref cmd) => { + command.push_str(cmd); + command.push_str(" "); + } + } + + // ===== Begin Python Args ===== + + command.push_str(" -text "); + command.push_str(&path_to_string(args.input_text)); + + command.push_str(" -use_prompt_name "); + command.push_str(&path_to_string(args.input_embedding)); + + command.push_str(" -output_file_name "); + command.push_str(&path_to_string(args.output_file_name)); + + // ===== End Python Args ===== + + if let Some(docker_options) = self.maybe_docker_options.as_ref() { + command = docker_options.to_command_string(&command); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + let mut env_vars = Vec::new(); + + // Copy all environment variables from the parent process. + // This is necessary to send all the kubernetes settings for Nvidia / CUDA. + for (env_key, env_value) in env::vars() { + if IGNORED_ENVIRONMENT_VARS.contains(&env_key) { + continue; + } + env_vars.push(( + OsString::from(env_key), + OsString::from(env_value), + )); + } + + let mut config = PopenConfig::default(); + + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + + if !env_vars.is_empty() { + config.env = Some(env_vars); + } + + let mut p = Popen::create(&command_parts, config)?; + + info!("Subprocess PID: {:?}", p.pid()); + + match self.maybe_execution_timeout { + None => { + let exit_status = p.wait()?; + info!("Subprocess exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + Some(timeout) => { + info!("Executing with timeout: {:?}", &timeout); + let exit_status = p.wait_timeout(timeout.clone())?; + + match exit_status { + None => { + // NB: If the program didn't successfully terminate, kill it. + info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout); + let _r = p.terminate()?; + Ok(CommandExitStatus::Timeout) + } + Some(exit_status) => { + info!("Subprocess timed wait exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + } + } + } + } +} + +#[derive(Clone)] pub struct VallEXCreateEmbeddingCommand { - + /// Where the code lives + root_code_directory: PathBuf, + /// A single executable script or a much larger bash command. + executable_or_command: ExecutableOrCommand, + /// eg. `source python/bin/activate` + maybe_virtual_env_activation_command: Option, + /// Optional default config file to use + maybe_default_config_path: Option, + /// If this is run under Docker (eg. in development), these are the options. + maybe_docker_options: Option, + /// If the execution should be ended after a certain point. + maybe_execution_timeout: Option, + /// Inference arg. + /// --checkpoint_dir: optional location for checkpoints directory + pub alternate_checkpoint_dir: Option } -// pub impl VallEXCreateEmbeddingCommand { -// pub fn new>( -// root_code_directory: P, -// executable_or_command: ExecutableOrCommand, -// maybe_virtual_env_activation_command: Option<&str>, -// maybe_default_config_path: Option

, -// maybe_docker_options: Option, -// maybe_execution_timeout: Option, -// alternate_checkpoint_dir: Option -// ) -> Self { -// Self { -// root_code_directory: root_code_directory.as_ref().to_path_buf(), -// executable_or_command, -// maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), -// maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), -// maybe_docker_options, -// maybe_execution_timeout, -// alternate_checkpoint_dir, -// } -// } -// } \ No newline at end of file +impl VallEXCreateEmbeddingCommand { + + pub fn new>( + root_code_directory: P, + executable_or_command: ExecutableOrCommand, + maybe_virtual_env_activation_command: Option<&str>, + maybe_default_config_path: Option

, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + alternate_checkpoint_dir: Option + ) -> Self { + Self { + root_code_directory: root_code_directory.as_ref().to_path_buf(), + executable_or_command, + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + } + } + + // helper funciton to set up env + pub fn from_env() -> AnyhowResult { + // TODO: fix for now just for compilation + let root_code_directory = easyenv::get_env_pathbuf_required( + "VALL_E_X_INFERENCE_ROOT_DIRECTORY")?; + + let maybe_inference_command = easyenv::get_env_string_optional( + "VALL_E_X_INFERENCE_COMMAND"); + + // EVENTUALLY should remove the check + // Optional, eg. `./infer.py`. Typically we'll use the command form instead. + let maybe_inference_executable = easyenv::get_env_pathbuf_optional( + "VALL_E_X_INFERENCE_EXECUTABLE"); + + let executable_or_command = match maybe_inference_command { + Some(command) => ExecutableOrCommand::Command(command), + None => match maybe_inference_executable { + Some(executable) => ExecutableOrCommand::Executable(executable), + None => return Err(anyhow!("neither command nor executable passed")), + }, + }; + + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( + "VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND"); + + let maybe_default_config_path = easyenv::get_env_pathbuf_optional( + "VALL_E_X_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH"); + + let maybe_execution_timeout = + easyenv::get_env_duration_seconds_optional("TIMEOUT_SECONDS"); + + + // Probably for local + let maybe_docker_options = easyenv::get_env_string_optional( + "VALL_E_X_INFERENCE_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_environment_variables: None, + maybe_gpu: Some(DockerGpu::All), + } + }); + + // Override for --checkpoint_dir at inference time + let alternate_checkpoint_dir = easyenv::get_env_pathbuf_optional( + "SAD_TALKER_ALTERNATE_CHECKPOINT_PATH"); + + Ok(Self { + root_code_directory, + executable_or_command, + maybe_virtual_env_activation_command, + maybe_default_config_path, + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + }) + } + +} diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 9f87447404..47f4320940 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -22,6 +22,7 @@ use newrelic_telemetry::Client as NewRelicClient; use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; +use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; @@ -205,6 +206,7 @@ pub struct SadTalkerDetails { // TODO: we will probably want a command type of some kind that implements // some kind of interface that we can just pass strings to. pub struct VallEXDetails { + pub downloaders: VallEXDownloaders, pub inference_command: VallEXInferenceCommand, pub create_embedding_command: VallEXCreateEmbeddingCommand } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index f25760e7f0..676145f5ad 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -316,8 +316,8 @@ async fn main() -> AnyhowResult<()> { }, vall_e_x: VallEXDetails { downloaders: VallEXDownloaders::build_all_from_env(), - inference_command: VallEXInferenceCommand, - create_embedding_command: VallEXCreateEmbeddingCommand, + inference_command: VallEXInferenceCommand::from_env()?, + create_embedding_command: VallEXCreateEmbeddingCommand::from_env()?, }, vits: VitsDetails { inference_command: vits_inference_command()?, From 3012dea984ee211ffcbb2aa4485932658989942a Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 17:58:07 -0400 Subject: [PATCH 16/19] process_single_tts_job augmented... --- .../job_types/tts/process_single_tts_job.rs | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 33f781b6e1..934fb27410 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -1,5 +1,6 @@ use anyhow::anyhow; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use enums::by_table::tts_models::tts_model_type::TtsModelType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::get_tts_model_for_inference_improved; @@ -60,46 +61,51 @@ pub async fn process_single_tts_job(job_dependencies: &JobDependencies, job: &Av // // info!("Lock acquired for job: {}", job.inference_job_token); + // Look for the Zeroshot job type + // let job_success_result = match job.maybe_model_type { + // Some(InferenceModelType::VallEX) => { + + // } + // Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), + // None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), + // }; -// Look for the Zeroshot job type + if let Some(model_type) = job.maybe_model_type { + if model_type == InferenceModelType::VallEX { + let job_success_result = vall_e_x::process_job::process_job(VALLEXProcessJobArgs { + job_dependencies, + job, + }).await?; + Ok(job_success_result) + }; + // do nothing... + }; - let job_success_result = match job.maybe_model_type { - Some(InferenceModelType::VallEX) => { - vall_e_x::process_job::process_job(VALLEXProcessJobArgs { - job_dependencies, - job, - }).await? - } - Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), - None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), - }; + // lets run the regular tts + let tts_model = match maybe_tts_model { + None => return Err(ProcessSingleJobError::Other(anyhow!("tts model not found: {:?}", tts_model_token))), + Some(model) => model, + }; -// let these run anyways ? - - let tts_model = match maybe_tts_model { - None => return Err(ProcessSingleJobError::Other(anyhow!("tts model not found: {:?}", tts_model_token))), - Some(model) => model, - }; - - let job_success_result: JobSuccessResult = match tts_model.tts_model_type { - TtsModelType::Tacotron2 => { - tacotron2_v2_early_fakeyou::process_job::process_job(ProcessJobArgs { - job_dependencies, - job, - tts_model: &tts_model, - raw_inference_text, - }).await? - } - TtsModelType::Vits => { - vits::process_job::process_job(VitsProcessJobArgs { - job_dependencies, - job, - tts_model: &tts_model, - raw_inference_text, - }).await? - } - }; - - Ok(job_success_result) + let job_success_result: JobSuccessResult = match tts_model.tts_model_type { + TtsModelType::Tacotron2 => { + tacotron2_v2_early_fakeyou::process_job::process_job(ProcessJobArgs { + job_dependencies, + job, + tts_model: &tts_model, + raw_inference_text, + }).await? + } + TtsModelType::Vits => { + vits::process_job::process_job(VitsProcessJobArgs { + job_dependencies, + job, + tts_model: &tts_model, + raw_inference_text, + }).await? + } + }; + Ok(job_success_result) + } } From 95888e683d101911c28dd24c01fd47ffe21058fb Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 18:01:25 -0400 Subject: [PATCH 17/19] Formatter works now, braces can get properly placed --- .../job_types/tts/process_single_tts_job.rs | 158 +++++++++--------- 1 file changed, 82 insertions(+), 76 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 934fb27410..8781ac8799 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -8,7 +8,7 @@ use tokens::tokens::tts_models::TtsModelToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; -use crate::job::job_types::tts::{tacotron2_v2_early_fakeyou, vits}; +use crate::job::job_types::tts::{ tacotron2_v2_early_fakeyou, vits }; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::process_job::ProcessJobArgs; use crate::job::job_types::tts::vits::process_job::VitsProcessJobArgs; use crate::job::job_types::tts::vall_e_x::process_job::VALLEXProcessJobArgs; @@ -16,96 +16,102 @@ use crate::job_dependencies::JobDependencies; use super::vall_e_x; -pub async fn process_single_tts_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { +pub async fn process_single_tts_job( + job_dependencies: &JobDependencies, + job: &AvailableInferenceJob +) -> Result { + let tts_model_token = match job.maybe_model_token.as_deref() { + None => { + return Err(ProcessSingleJobError::Other(anyhow!("no model token on job"))); + } + Some(model_token) => TtsModelToken::new_from_str(model_token), + }; - let tts_model_token = match job.maybe_model_token.as_deref() { - None => return Err(ProcessSingleJobError::Other(anyhow!("no model token on job"))), - Some(model_token) => TtsModelToken::new_from_str(model_token), - }; + // TODO: Move common checks for slurs, etc. here. - // TODO: Move common checks for slurs, etc. here. + let raw_inference_text = job.maybe_raw_inference_text + .as_deref() + .ok_or(ProcessSingleJobError::Other(anyhow!("no inference text")))?; - let raw_inference_text = job.maybe_raw_inference_text - .as_deref() - .ok_or(ProcessSingleJobError::Other(anyhow!("no inference text")))?; + // TODO: Interrogate cache (which also depends on other flags) + let maybe_tts_model = get_tts_model_for_inference_improved( + &job_dependencies.mysql_pool, + tts_model_token.as_str() + ).await.map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?; - // TODO: Interrogate cache (which also depends on other flags) - let maybe_tts_model = get_tts_model_for_inference_improved( - &job_dependencies.mysql_pool, tts_model_token.as_str()) - .await - .map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?; + // TODO: Attempt to grab job lock -// TODO: Attempt to grab job lock + // // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // + // + // info!("Attempting to grab lock for job: {}", job.inference_job_token); + // + // let lock_acquired = + // mark_tts_inference_job_pending_and_grab_lock(&job_args.mysql_pool, job.id) + // .await + // .map_err(|e| ProcessSingleJobError::Other(e))?; + // + // if !lock_acquired { + // warn!("Could not acquire job lock for: {:?}", &job.id); + // let duration = start.elapsed(); + // + // since_creation_span.set_attribute("status", "failure"); + // since_creation_span.set_duration(duration); + // + // job_iteration_span.set_attribute("status", "failure"); + // job_iteration_span.set_duration(duration); + // + // return Ok((since_creation_span, job_iteration_span)); + // } + // + // info!("Lock acquired for job: {}", job.inference_job_token); -// // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // -// -// info!("Attempting to grab lock for job: {}", job.inference_job_token); -// -// let lock_acquired = -// mark_tts_inference_job_pending_and_grab_lock(&job_args.mysql_pool, job.id) -// .await -// .map_err(|e| ProcessSingleJobError::Other(e))?; -// -// if !lock_acquired { -// warn!("Could not acquire job lock for: {:?}", &job.id); -// let duration = start.elapsed(); -// -// since_creation_span.set_attribute("status", "failure"); -// since_creation_span.set_duration(duration); -// -// job_iteration_span.set_attribute("status", "failure"); -// job_iteration_span.set_duration(duration); -// -// return Ok((since_creation_span, job_iteration_span)); -// } -// -// info!("Lock acquired for job: {}", job.inference_job_token); + // Look for the Zeroshot job type + // let job_success_result = match job.maybe_model_type { + // Some(InferenceModelType::VallEX) => { - // Look for the Zeroshot job type - // let job_success_result = match job.maybe_model_type { - // Some(InferenceModelType::VallEX) => { - - // } - // Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), - // None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), - // }; + // } + // Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), + // None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), + // }; if let Some(model_type) = job.maybe_model_type { - if model_type == InferenceModelType::VallEX { - let job_success_result = vall_e_x::process_job::process_job(VALLEXProcessJobArgs { - job_dependencies, - job, - }).await?; - Ok(job_success_result) - }; - // do nothing... - }; + if model_type == InferenceModelType::VallEX { + let job_success_result = vall_e_x::process_job::process_job(VALLEXProcessJobArgs { + job_dependencies, + job, + }).await?; + Ok(job_success_result); + } + // do nothing try regular tts else other zero shot models + } // lets run the regular tts let tts_model = match maybe_tts_model { - None => return Err(ProcessSingleJobError::Other(anyhow!("tts model not found: {:?}", tts_model_token))), - Some(model) => model, + None => { + return Err( + ProcessSingleJobError::Other(anyhow!("tts model not found: {:?}", tts_model_token)) + ); + } + Some(model) => model, }; let job_success_result: JobSuccessResult = match tts_model.tts_model_type { - TtsModelType::Tacotron2 => { - tacotron2_v2_early_fakeyou::process_job::process_job(ProcessJobArgs { - job_dependencies, - job, - tts_model: &tts_model, - raw_inference_text, - }).await? - } - TtsModelType::Vits => { - vits::process_job::process_job(VitsProcessJobArgs { - job_dependencies, - job, - tts_model: &tts_model, - raw_inference_text, - }).await? - } + TtsModelType::Tacotron2 => { + tacotron2_v2_early_fakeyou::process_job::process_job(ProcessJobArgs { + job_dependencies, + job, + tts_model: &tts_model, + raw_inference_text, + }).await? + } + TtsModelType::Vits => { + vits::process_job::process_job(VitsProcessJobArgs { + job_dependencies, + job, + tts_model: &tts_model, + raw_inference_text, + }).await? + } }; Ok(job_success_result) - } } - From 13809ad309aa3fc49e5adf94fb1b077a5e90ba62 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 18:04:46 -0400 Subject: [PATCH 18/19] Fixed issue --- .../src/job/job_types/tts/process_single_tts_job.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 8781ac8799..2a4fc2b92a 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -20,6 +20,7 @@ pub async fn process_single_tts_job( job_dependencies: &JobDependencies, job: &AvailableInferenceJob ) -> Result { + let tts_model_token = match job.maybe_model_token.as_deref() { None => { return Err(ProcessSingleJobError::Other(anyhow!("no model token on job"))); @@ -28,7 +29,6 @@ pub async fn process_single_tts_job( }; // TODO: Move common checks for slurs, etc. here. - let raw_inference_text = job.maybe_raw_inference_text .as_deref() .ok_or(ProcessSingleJobError::Other(anyhow!("no inference text")))?; @@ -40,7 +40,6 @@ pub async fn process_single_tts_job( ).await.map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?; // TODO: Attempt to grab job lock - // // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // // // info!("Attempting to grab lock for job: {}", job.inference_job_token); @@ -114,4 +113,5 @@ pub async fn process_single_tts_job( } }; Ok(job_success_result) + } From d8c4df4c0255c919fde79d60705e3ca7a105f9b9 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Fri, 6 Oct 2023 18:12:04 -0400 Subject: [PATCH 19/19] Ok fixed the odd linting error. --- .../src/job/job_types/tts/process_single_tts_job.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 2a4fc2b92a..58bcffa96f 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -79,7 +79,7 @@ pub async fn process_single_tts_job( job_dependencies, job, }).await?; - Ok(job_success_result); + Ok::(job_success_result); } // do nothing try regular tts else other zero shot models }