From 2da81d47a7b4c4689695228583affe052ef81de4 Mon Sep 17 00:00:00 2001 From: Kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Fri, 18 Oct 2024 21:06:16 +0530 Subject: [PATCH] sigh, add an f5-tts inference_category --- .../inference_category.rs | 12 +++++++++++- .../web/batch_get_inference_job_status.rs | 4 +++- .../web/get_inference_job_status.rs | 4 +++- .../web/list_session_jobs.rs | 4 +++- .../batch_get_inference_job_status_handler.rs | 3 ++- .../get/get_inference_job_status_handler.rs | 3 ++- .../list/list_session_jobs_handler.rs | 3 ++- .../utils/estimates/estimate_job_progress.rs | 3 +++ .../enqueue_infer_f5_tts_handler.rs | 19 +------------------ 9 files changed, 30 insertions(+), 25 deletions(-) diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index efed24461e..710993bea9 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -67,6 +67,9 @@ pub enum InferenceCategory { #[deprecated(note = "This was for Bevy engine's server side rendering.")] #[serde(rename = "convert_bvh_to_workflow")] ConvertBvhToWorkflow, + + #[serde(rename = "f5_tts")] + F5TTS, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -85,6 +88,7 @@ impl InferenceCategory { Self::ImageGeneration => "image_generation", Self::Mocap => "mocap", Self::Workflow => "workflow", + Self::F5TTS => "f5_tts", Self::FormatConversion => "format_conversion", Self::LivePortrait => "live_portrait", Self::VideoFilter => "video_filter", @@ -99,6 +103,7 @@ impl InferenceCategory { "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), "image_generation" => Ok(Self::ImageGeneration), + "f5_tts" => Ok(Self::F5TTS), "mocap" => Ok(Self::Mocap), "workflow" => Ok(Self::Workflow), "format_conversion" => Ok(Self::FormatConversion), @@ -119,6 +124,7 @@ impl InferenceCategory { Self::VoiceConversion, Self::ImageGeneration, Self::Mocap, + Self::F5TTS, Self::Workflow, Self::FormatConversion, Self::LivePortrait, @@ -144,6 +150,7 @@ mod tests { assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); assert_serialization(InferenceCategory::ImageGeneration, "image_generation"); assert_serialization(InferenceCategory::Mocap, "mocap"); + assert_serialization(InferenceCategory::F5TTS, "f5_tts"); assert_serialization(InferenceCategory::Workflow, "workflow"); assert_serialization(InferenceCategory::FormatConversion, "format_conversion"); assert_serialization(InferenceCategory::LivePortrait, "live_portrait"); @@ -158,6 +165,7 @@ mod tests { assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(InferenceCategory::ImageGeneration.to_str(), "image_generation"); + assert_eq!(InferenceCategory::F5TTS.to_str(), "f5_tts"); assert_eq!(InferenceCategory::Mocap.to_str(), "mocap"); assert_eq!(InferenceCategory::Workflow.to_str(), "workflow"); assert_eq!(InferenceCategory::FormatConversion.to_str(), "format_conversion"); @@ -173,6 +181,7 @@ mod tests { assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); assert_eq!(InferenceCategory::from_str("image_generation").unwrap(), InferenceCategory::ImageGeneration); + assert_eq!(InferenceCategory::from_str("f5_tts").unwrap(), InferenceCategory::F5TTS); assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap); assert_eq!(InferenceCategory::from_str("workflow").unwrap(), InferenceCategory::Workflow); assert_eq!(InferenceCategory::from_str("format_conversion").unwrap(), InferenceCategory::FormatConversion); @@ -185,12 +194,13 @@ mod tests { fn all_variants() { // Static check let mut variants = InferenceCategory::all_variants(); - assert_eq!(variants.len(), 11); + assert_eq!(variants.len(), 12); assert_eq!(variants.pop_first(), Some(InferenceCategory::DeprecatedField)); assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(InferenceCategory::ImageGeneration)); + assert_eq!(variants.pop_first(), Some(InferenceCategory::F5TTS)); assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap)); assert_eq!(variants.pop_first(), Some(InferenceCategory::Workflow)); assert_eq!(variants.pop_first(), Some(InferenceCategory::FormatConversion)); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs index 49730a244c..044d6b1181 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; -use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; use sqlx::mysql::MySqlRow; use sqlx::pool::PoolConnection; +use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; @@ -178,6 +178,7 @@ fn raw_records_to_public_result(records: Vec) -> V InferenceCategory::Workflow => Some("Workflow"), InferenceCategory::FormatConversion => Some("format conversion"), InferenceCategory::LivePortrait => Some("Live Portrait"), + InferenceCategory::F5TTS => Some("F5 TTS"), InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"), InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix }; @@ -188,6 +189,7 @@ fn raw_records_to_public_result(records: Vec) -> V let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category { InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), + InferenceCategory::F5TTS => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 2b781fa96a..01245dc624 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -1,8 +1,8 @@ use anyhow::anyhow; use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use log::warn; -use sqlx::{MySql, MySqlPool}; use sqlx::pool::PoolConnection; +use sqlx::{MySql, MySqlPool}; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; @@ -154,6 +154,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::Workflow => Some("Workflow"), InferenceCategory::FormatConversion => Some("format conversion"), InferenceCategory::LivePortrait => Some("Live Portrait"), + InferenceCategory::F5TTS => Some("F5 TTS"), InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"), InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix }; @@ -164,6 +165,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category { InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), + InferenceCategory::F5TTS => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs index 270a4d49a1..e5ad9509a8 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs @@ -2,9 +2,9 @@ use std::collections::HashSet; use anyhow::anyhow; use chrono::{DateTime, Utc}; -use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; use sqlx::mysql::MySqlRow; use sqlx::pool::PoolConnection; +use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; @@ -217,6 +217,7 @@ fn raw_records_to_public_result(records: Vec) -> V maybe_model_title = match record.inference_category { InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), + InferenceCategory::F5TTS => Some("F5 TTS"), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), InferenceCategory::VideoFilter => Some("Video Filter"), InferenceCategory::ImageGeneration => Some("Image Generation"), @@ -234,6 +235,7 @@ fn raw_records_to_public_result(records: Vec) -> V let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category { InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), + InferenceCategory::F5TTS => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/batch_get_inference_job_status_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/batch_get_inference_job_status_handler.rs index 46024d6d64..0831c30c6d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/batch_get_inference_job_status_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/batch_get_inference_job_status_handler.rs @@ -346,7 +346,8 @@ fn db_record_to_response_payload( .get_full_object_path_str() .to_string() } - InferenceCategory::TextToSpeech => { + InferenceCategory::TextToSpeech | + InferenceCategory::F5TTS => { match result_details.entity_type.as_str() { "media_file" => { // NB: We're migrating TTS to media_files. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/get_inference_job_status_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/get_inference_job_status_handler.rs index 3d41af8d20..39acfe4308 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/get_inference_job_status_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/get_inference_job_status_handler.rs @@ -337,7 +337,8 @@ fn record_to_payload( .get_full_object_path_str() .to_string() } - InferenceCategory::TextToSpeech => { + InferenceCategory::TextToSpeech + | InferenceCategory::F5TTS => { match result_details.entity_type.as_str() { "media_file" => { // NB: We're migrating TTS to media_files. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/list/list_session_jobs_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/list/list_session_jobs_handler.rs index 7120b84359..03fb17f368 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/list/list_session_jobs_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/list/list_session_jobs_handler.rs @@ -429,7 +429,8 @@ fn db_record_to_response_payload( .get_full_object_path_str() .to_string() } - InferenceCategory::TextToSpeech => { + InferenceCategory::TextToSpeech | + InferenceCategory::F5TTS => { match result_details.entity_type.as_str() { "media_file" => { // NB: We're migrating TTS to media_files. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs index 0a83235daa..649884521c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs @@ -21,6 +21,8 @@ const VID_FACE_FUSION_AVERAGE_SECONDS : u64 = 45; /// Lipsync with SadTalker (super old and slow) const LIPSYNC_JOB_AVERAGE_SECONDS : u64 = 60 * 3; +const F5_TTS_JOB_AVERAGE_SECONDS : u64 = 12; + pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option<&PolymorphicInferenceArgs>) -> u8 { match job.status { // Jobs that haven't started @@ -61,6 +63,7 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option InferenceCategory::TextToSpeech => percent(duration_seconds, TTS_JOB_AVERAGE_SECONDS), InferenceCategory::VoiceConversion => percent(duration_seconds, VC_JOB_AVERAGE_SECONDS), InferenceCategory::LivePortrait => percent(duration_seconds, LIVE_PORTRAIT_JOB_AVERAGE_SECONDS), + InferenceCategory::F5TTS => percent(duration_seconds, F5_TTS_JOB_AVERAGE_SECONDS), // TODO: Better estimate using video duration, params, etc. InferenceCategory::Workflow => comfy_workflow_estimate(maybe_args, duration_seconds), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_f5_tts_handler/enqueue_infer_f5_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_f5_tts_handler/enqueue_infer_f5_tts_handler.rs index 966f3df56c..6c8b4f027f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_f5_tts_handler/enqueue_infer_f5_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_f5_tts_handler/enqueue_infer_f5_tts_handler.rs @@ -354,23 +354,6 @@ pub async fn enqueue_infer_f5_tts_handler( "input token is not a media_upload or media_file token".to_string())); }; - // let maybe_inference_args = match tts_model.job_type() { - // InferenceJobType::Tacotron2 => Some(GenericInferenceArgs { - // inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), - // args: None, // NB: We don't need to encode any args yet for TT2. - // }), - // InferenceJobType::GptSovits => Some(GenericInferenceArgs { - // inference_category: Some(InferenceCategoryAbbreviated::GptSovits), - // args: Some(PolymorphicInferenceArgs::Gs(GptSovitsPayload { - // append_advertisement: None, - // // Model download only args: - // creator_visibility: None, - // maybe_title: None, - // maybe_description: None, - // })), - // }), - // _ => None, // NB: Shouldn't occur - // }; let maybe_product_category = Some(InferenceJobProductCategory::TtsF5); @@ -378,7 +361,7 @@ pub async fn enqueue_infer_f5_tts_handler( uuid_idempotency_token: &request.uuid_idempotency_token, job_type: InferenceJobType::F5TTS, maybe_product_category, - inference_category: InferenceCategory::TextToSpeech, + inference_category: InferenceCategory::F5TTS, maybe_model_type, maybe_model_token: None, maybe_input_source_token: Some(&media_token),