sigh, add an f5-tts inference_category

crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs

@@ -67,6 +67,9 @@ pub enum InferenceCategory {
   #[deprecated(note = "This was for Bevy engine's server side rendering.")]
   #[serde(rename = "convert_bvh_to_workflow")]
   ConvertBvhToWorkflow,
+
+  #[serde(rename = "f5_tts")]
+  F5TTS,
 }
 
 // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -85,6 +88,7 @@ impl InferenceCategory {
       Self::ImageGeneration => "image_generation",
       Self::Mocap => "mocap",
       Self::Workflow => "workflow",
+      Self::F5TTS => "f5_tts",
       Self::FormatConversion => "format_conversion",
       Self::LivePortrait => "live_portrait",
       Self::VideoFilter => "video_filter",
@@ -99,6 +103,7 @@ impl InferenceCategory {
       "text_to_speech" => Ok(Self::TextToSpeech),
       "voice_conversion" => Ok(Self::VoiceConversion),
       "image_generation" => Ok(Self::ImageGeneration),
+      "f5_tts" => Ok(Self::F5TTS),
       "mocap" => Ok(Self::Mocap),
       "workflow" => Ok(Self::Workflow),
       "format_conversion" => Ok(Self::FormatConversion),
@@ -119,6 +124,7 @@ impl InferenceCategory {
       Self::VoiceConversion,
       Self::ImageGeneration,
       Self::Mocap,
+      Self::F5TTS,
       Self::Workflow,
       Self::FormatConversion,
       Self::LivePortrait,
@@ -144,6 +150,7 @@ mod tests {
       assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion");
       assert_serialization(InferenceCategory::ImageGeneration, "image_generation");
       assert_serialization(InferenceCategory::Mocap, "mocap");
+      assert_serialization(InferenceCategory::F5TTS, "f5_tts");
       assert_serialization(InferenceCategory::Workflow, "workflow");
       assert_serialization(InferenceCategory::FormatConversion, "format_conversion");
       assert_serialization(InferenceCategory::LivePortrait, "live_portrait");
@@ -158,6 +165,7 @@ mod tests {
       assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech");
       assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion");
       assert_eq!(InferenceCategory::ImageGeneration.to_str(), "image_generation");
+      assert_eq!(InferenceCategory::F5TTS.to_str(), "f5_tts");
       assert_eq!(InferenceCategory::Mocap.to_str(), "mocap");
       assert_eq!(InferenceCategory::Workflow.to_str(), "workflow");
       assert_eq!(InferenceCategory::FormatConversion.to_str(), "format_conversion");
@@ -173,6 +181,7 @@ mod tests {
       assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech);
       assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion);
       assert_eq!(InferenceCategory::from_str("image_generation").unwrap(), InferenceCategory::ImageGeneration);
+      assert_eq!(InferenceCategory::from_str("f5_tts").unwrap(), InferenceCategory::F5TTS);
       assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap);
       assert_eq!(InferenceCategory::from_str("workflow").unwrap(), InferenceCategory::Workflow);
       assert_eq!(InferenceCategory::from_str("format_conversion").unwrap(), InferenceCategory::FormatConversion);
@@ -185,12 +194,13 @@ mod tests {
     fn all_variants() {
       // Static check
       let mut variants = InferenceCategory::all_variants();
-      assert_eq!(variants.len(), 11);
+      assert_eq!(variants.len(), 12);
       assert_eq!(variants.pop_first(), Some(InferenceCategory::DeprecatedField));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::ImageGeneration));
+      assert_eq!(variants.pop_first(), Some(InferenceCategory::F5TTS));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::Workflow));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::FormatConversion));

crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs

@@ -1,8 +1,8 @@
 use anyhow::anyhow;
 use chrono::{DateTime, Utc};
-use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
 use sqlx::mysql::MySqlRow;
 use sqlx::pool::PoolConnection;
+use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
@@ -178,6 +178,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
             InferenceCategory::Workflow => Some("Workflow"),
             InferenceCategory::FormatConversion => Some("format conversion"),
             InferenceCategory::LivePortrait => Some("Live Portrait"),
+            InferenceCategory::F5TTS => Some("F5 TTS"),
             InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
             InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix
           };
@@ -188,6 +189,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
         let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category {
           InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
+          InferenceCategory::F5TTS => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
           InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -1,8 +1,8 @@
 use anyhow::anyhow;
 use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
 use log::warn;
-use sqlx::{MySql, MySqlPool};
 use sqlx::pool::PoolConnection;
+use sqlx::{MySql, MySqlPool};
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
@@ -154,6 +154,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
       InferenceCategory::Workflow => Some("Workflow"),
       InferenceCategory::FormatConversion => Some("format conversion"),
       InferenceCategory::LivePortrait => Some("Live Portrait"),
+      InferenceCategory::F5TTS => Some("F5 TTS"),
       InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
       InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix
     };
@@ -164,6 +165,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
   let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category {
     InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
+    InferenceCategory::F5TTS => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
     InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),

crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs

@@ -2,9 +2,9 @@ use std::collections::HashSet;
 
 use anyhow::anyhow;
 use chrono::{DateTime, Utc};
-use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
 use sqlx::mysql::MySqlRow;
 use sqlx::pool::PoolConnection;
+use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row};
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
@@ -217,6 +217,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
           maybe_model_title = match record.inference_category {
             InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
             InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
+            InferenceCategory::F5TTS => Some("F5 TTS"),
             InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
             InferenceCategory::VideoFilter => Some("Video Filter"),
             InferenceCategory::ImageGeneration => Some("Image Generation"),
@@ -234,6 +235,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
         let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category {
           InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
+          InferenceCategory::F5TTS => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
           InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::ImageGeneration => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/batch_get_inference_job_status_handler.rs

@@ -346,7 +346,8 @@ fn db_record_to_response_payload(
               .get_full_object_path_str()
               .to_string()
         }
-        InferenceCategory::TextToSpeech => {
+        InferenceCategory::TextToSpeech |
+        InferenceCategory::F5TTS => {
           match result_details.entity_type.as_str() {
             "media_file" => {
               // NB: We're migrating TTS to media_files.

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get/get_inference_job_status_handler.rs

@@ -337,7 +337,8 @@ fn record_to_payload(
               .get_full_object_path_str()
               .to_string()
         }
-        InferenceCategory::TextToSpeech => {
+        InferenceCategory::TextToSpeech
+        | InferenceCategory::F5TTS => {
           match result_details.entity_type.as_str() {
             "media_file" => {
               // NB: We're migrating TTS to media_files.

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/list/list_session_jobs_handler.rs

@@ -429,7 +429,8 @@ fn db_record_to_response_payload(
               .get_full_object_path_str()
               .to_string()
         }
-        InferenceCategory::TextToSpeech => {
+        InferenceCategory::TextToSpeech |
+        InferenceCategory::F5TTS => {
           match result_details.entity_type.as_str() {
             "media_file" => {
               // NB: We're migrating TTS to media_files.

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs

@@ -21,6 +21,8 @@ const VID_FACE_FUSION_AVERAGE_SECONDS : u64 = 45;
 /// Lipsync with SadTalker (super old and slow)
 const LIPSYNC_JOB_AVERAGE_SECONDS : u64 = 60 * 3;
 
+const F5_TTS_JOB_AVERAGE_SECONDS : u64 = 12;
+
 pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option<&PolymorphicInferenceArgs>) -> u8 {
   match job.status {
     // Jobs that haven't started
@@ -61,6 +63,7 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option
     InferenceCategory::TextToSpeech => percent(duration_seconds, TTS_JOB_AVERAGE_SECONDS),
     InferenceCategory::VoiceConversion => percent(duration_seconds, VC_JOB_AVERAGE_SECONDS),
     InferenceCategory::LivePortrait => percent(duration_seconds, LIVE_PORTRAIT_JOB_AVERAGE_SECONDS),
+    InferenceCategory::F5TTS => percent(duration_seconds, F5_TTS_JOB_AVERAGE_SECONDS),
 
     // TODO: Better estimate using video duration, params, etc.
     InferenceCategory::Workflow => comfy_workflow_estimate(maybe_args, duration_seconds),

crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_f5_tts_handler/enqueue_infer_f5_tts_handler.rs

@@ -354,23 +354,6 @@ pub async fn enqueue_infer_f5_tts_handler(
         "input token is not a media_upload or media_file token".to_string()));
     };
 
-  // let maybe_inference_args = match tts_model.job_type() {
-  //   InferenceJobType::Tacotron2 => Some(GenericInferenceArgs {
-  //     inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech),
-  //     args: None, // NB: We don't need to encode any args yet for TT2.
-  //   }),
-  //   InferenceJobType::GptSovits => Some(GenericInferenceArgs {
-  //     inference_category: Some(InferenceCategoryAbbreviated::GptSovits),
-  //     args: Some(PolymorphicInferenceArgs::Gs(GptSovitsPayload {
-  //       append_advertisement: None,
-  //       // Model download only args:
-  //       creator_visibility: None,
-  //       maybe_title: None,
-  //       maybe_description: None,
-  //     })),
-  //   }),
-  //   _ => None, // NB: Shouldn't occur
-  // };
 
   let maybe_product_category = Some(InferenceJobProductCategory::TtsF5);
 
@@ -378,7 +361,7 @@ pub async fn enqueue_infer_f5_tts_handler(
     uuid_idempotency_token: &request.uuid_idempotency_token,
     job_type: InferenceJobType::F5TTS,
     maybe_product_category,
-    inference_category: InferenceCategory::TextToSpeech,
+    inference_category: InferenceCategory::F5TTS,
     maybe_model_type,
     maybe_model_token: None,
     maybe_input_source_token: Some(&media_token),