other job types

.idea/dictionaries/bt.xml

@@ -26,6 +26,7 @@
       <w>hparams</w>
       <w>huggingface</w>
       <w>lipsync</w>
+      <w>mocap</w>
       <w>mochi</w>
       <w>nestedness</w>
       <w>pathbuf</w>

crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_type.rs

@@ -13,16 +13,45 @@ use strum::EnumIter;
 /// YOU CAN ADD NEW VALUES, BUT DO NOT CHANGE EXISTING VALUES WITHOUT A MIGRATION STRATEGY.
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)]
+#[serde(rename_all = "snake_case")]
 pub enum InferenceJobType {
+  /// Jobs that run ComfyUI workflows
+  ComfyUi,
+
   /// A job that turns "FBX" game engine files into "GLTF" files (Bevy-compatible).
   #[serde(rename = "convert_fbx_gltf")]
   ConvertFbxToGltf,
 
-  #[serde(rename = "rerender_a_video")]
+  /// Process a video into BVH mocap animation data for game engines
+  MocapNet,
+
+  /// Re-render a video is a video style transfer algorithm. We developed code
+  /// around it, but chose to develop AnimateDiff / ComfyUI support instead.
   RerenderAVideo,
 
+  /// RVC is a voice conversion model. RVCv2 is the most popular such model currently.
+  #[serde(rename = "rvc_v2")]
+  RvcV2,
+
+  /// SadTalker does image-to-video lip-syncing when given an audio file and image.
+  SadTalker,
+
+  /// so-vits-svc voice conversion. This predates RVCv2.
+  SoVitsSvc,
+
+  /// Stable diffusion image generation
+  StableDiffusion,
+
+  /// StyleTTS2 is a zero shot multi-speaker TTS model.
+  /// This job type should handle both speaker vector encoding and inference.
+  #[serde(rename = "styletts2")]
+  StyleTTS2,
+
+  /// TT2 Text to speech
+  Tacotron2,
+
   /// A value we may use in the future for historical jobs
-  #[serde(rename = "unknown")]
+  /// (i.e. when we backfill the database column and make it non-nullable)
   #[default]
   Unknown,
 }
@@ -35,15 +64,32 @@ impl_mysql_enum_coders!(InferenceJobType);
 impl InferenceJobType {
   pub fn to_str(&self) -> &'static str {
     match self {
+      Self::ComfyUi => "comfy_ui",
       Self::ConvertFbxToGltf => "convert_fbx_gltf",
+      Self::MocapNet => "mocap_net",
       Self::RerenderAVideo => "rerender_a_video",
+      Self::RvcV2 => "rvc_v2",
+      Self::SadTalker => "sad_talker",
+      Self::SoVitsSvc => "so_vits_svc",
+      Self::StableDiffusion => "stable_diffusion",
+      Self::StyleTTS2 => "styletts2",
+      Self::Tacotron2 => "tacotron2",
       Self::Unknown => "unknown",
     }
   }
 
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
+      "comfy_ui" => Ok(Self::ComfyUi),
       "convert_fbx_gltf" => Ok(Self::ConvertFbxToGltf),
+      "mocap_net" => Ok(Self::MocapNet),
+      "rerender_a_video" => Ok(Self::RerenderAVideo),
+      "rvc_v2" => Ok(Self::RvcV2),
+      "sad_talker" => Ok(Self::SadTalker),
+      "so_vits_svc" => Ok(Self::SoVitsSvc),
+      "stable_diffusion" => Ok(Self::StableDiffusion),
+      "styletts2" => Ok(Self::StyleTTS2),
+      "tacotron2" => Ok(Self::Tacotron2),
       "unknown" => Ok(Self::Unknown),
       _ => Err(format!("invalid value: {:?}", value)),
     }
@@ -53,7 +99,16 @@ impl InferenceJobType {
     // NB: BTreeSet is sorted
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
+      Self::ComfyUi,
       Self::ConvertFbxToGltf,
+      Self::MocapNet,
+      Self::RerenderAVideo,
+      Self::RvcV2,
+      Self::SadTalker,
+      Self::SoVitsSvc,
+      Self::StableDiffusion,
+      Self::StyleTTS2,
+      Self::Tacotron2,
       Self::Unknown,
     ])
   }
@@ -74,22 +129,46 @@ mod tests {
 
     #[test]
     fn test_serialization() {
+      assert_serialization(InferenceJobType::ComfyUi, "comfy_ui");
       assert_serialization(InferenceJobType::ConvertFbxToGltf, "convert_fbx_gltf");
+      assert_serialization(InferenceJobType::MocapNet, "mocap_net");
       assert_serialization(InferenceJobType::RerenderAVideo, "rerender_a_video");
+      assert_serialization(InferenceJobType::RvcV2, "rvc_v2");
+      assert_serialization(InferenceJobType::SadTalker, "sad_talker");
+      assert_serialization(InferenceJobType::SoVitsSvc, "so_vits_svc");
+      assert_serialization(InferenceJobType::StableDiffusion, "stable_diffusion");
+      assert_serialization(InferenceJobType::StyleTTS2, "styletts2");
+      assert_serialization(InferenceJobType::Tacotron2, "tacotron2");
       assert_serialization(InferenceJobType::Unknown, "unknown");
     }
 
     #[test]
     fn to_str() {
+      assert_eq!(InferenceJobType::ComfyUi.to_str(), "comfy_ui");
       assert_eq!(InferenceJobType::ConvertFbxToGltf.to_str(), "convert_fbx_gltf");
+      assert_eq!(InferenceJobType::MocapNet.to_str(), "mocap_net");
       assert_eq!(InferenceJobType::RerenderAVideo.to_str(), "rerender_a_video");
+      assert_eq!(InferenceJobType::RvcV2.to_str(), "rvc_v2");
+      assert_eq!(InferenceJobType::SadTalker.to_str(), "sad_talker");
+      assert_eq!(InferenceJobType::SoVitsSvc.to_str(), "so_vits_svc");
+      assert_eq!(InferenceJobType::StableDiffusion.to_str(), "stable_diffusion");
+      assert_eq!(InferenceJobType::StyleTTS2.to_str(), "styletts2");
+      assert_eq!(InferenceJobType::Tacotron2.to_str(), "tacotron2");
       assert_eq!(InferenceJobType::Unknown.to_str(), "unknown");
     }
 
     #[test]
     fn from_str() {
+      assert_eq!(InferenceJobType::from_str("comfy_ui").unwrap(), InferenceJobType::ComfyUi);
       assert_eq!(InferenceJobType::from_str("convert_fbx_gltf").unwrap(), InferenceJobType::ConvertFbxToGltf);
+      assert_eq!(InferenceJobType::from_str("mocap_net").unwrap(), InferenceJobType::MocapNet);
       assert_eq!(InferenceJobType::from_str("rerender_a_video").unwrap(), InferenceJobType::RerenderAVideo);
+      assert_eq!(InferenceJobType::from_str("rvc_v2").unwrap(), InferenceJobType::RvcV2);
+      assert_eq!(InferenceJobType::from_str("sad_talker").unwrap(), InferenceJobType::SadTalker);
+      assert_eq!(InferenceJobType::from_str("so_vits_svc").unwrap(), InferenceJobType::SoVitsSvc);
+      assert_eq!(InferenceJobType::from_str("stable_diffusion").unwrap(), InferenceJobType::StableDiffusion);
+      assert_eq!(InferenceJobType::from_str("styletts2").unwrap(), InferenceJobType::StyleTTS2);
+      assert_eq!(InferenceJobType::from_str("tacotron2").unwrap(), InferenceJobType::Tacotron2);
       assert_eq!(InferenceJobType::from_str("unknown").unwrap(), InferenceJobType::Unknown);
     }
 
@@ -97,9 +176,17 @@ mod tests {
     fn all_variants() {
       // Static check
       let mut variants = InferenceJobType::all_variants();
-      assert_eq!(variants.len(), 3);
+      assert_eq!(variants.len(), 11);
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::ComfyUi));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::ConvertFbxToGltf));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::MocapNet));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::RerenderAVideo));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::RvcV2));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::SadTalker));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::SoVitsSvc));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::StableDiffusion));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::StyleTTS2));
+      assert_eq!(variants.pop_first(), Some(InferenceJobType::Tacotron2));
       assert_eq!(variants.pop_first(), Some(InferenceJobType::Unknown));
       assert_eq!(variants.pop_first(), None);
 

crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_face_animation.rs

@@ -308,7 +308,7 @@ pub async fn enqueue_face_animation_handler(
 
   let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
     uuid_idempotency_token: &request.uuid_idempotency_token,
-    job_type: InferenceJobType::Unknown,
+    job_type: InferenceJobType::SadTalker,
     inference_category: InferenceCategory::LipsyncAnimation,
     maybe_model_type: Some(InferenceModelType::SadTalker), // NB: Model is static during inference
     maybe_model_token: None, // NB: Model is static during inference

crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs

@@ -407,7 +407,7 @@ pub async fn enqueue_image_generation_request(
     // create the job record here!
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
         uuid_idempotency_token: &request.uuid_idempotency_token,
-        job_type: InferenceJobType::Unknown,
+        job_type: InferenceJobType::StableDiffusion,
         inference_category: InferenceCategory::ImageGeneration,
         maybe_model_type: Some(InferenceModelType::StableDiffusion), // NB: Model is static during inference
         maybe_model_token: None, // NB: Model is static during inference

crates/service/web/storyteller_web/src/http_server/endpoints/mocap/enqueue_mocapnet.rs

@@ -188,7 +188,7 @@ pub async fn enqueue_mocapnet_handler(
 
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
         uuid_idempotency_token: &request.uuid_idempotency_token,
-        job_type: InferenceJobType::Unknown,
+        job_type: InferenceJobType::MocapNet,
         inference_category: InferenceCategory::Mocap,
         maybe_model_type: Some(InferenceModelType::MocapNet), // NB: Model is static during inference
         maybe_model_token: None, // NB: Model is static during inference

crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs

@@ -398,7 +398,7 @@ pub async fn enqueue_infer_tts_handler(
 
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
       uuid_idempotency_token: &request.uuid_idempotency_token,
-      job_type: InferenceJobType::Unknown,
+      job_type: InferenceJobType::Tacotron2,
       inference_category: InferenceCategory::TextToSpeech,
       maybe_model_type: Some(model_type),
       maybe_model_token: Some(request.tts_model_token.as_str()),

crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs

@@ -290,9 +290,19 @@ pub async fn enqueue_voice_conversion_inference_handler(
     });
   }
 
+  let job_type = match model_inference_info.job_model_type {
+    InferenceModelType::SoVitsSvc => InferenceJobType::SoVitsSvc,
+    InferenceModelType::RvcV2 => InferenceJobType::RvcV2,
+    _ => {
+      // In theory, this shouldn't catch anything.
+      error!("wrong model type for voice conversion: {:?}", model_inference_info.job_model_type);
+      return Err(EnqueueVoiceConversionInferenceError::ServerError)
+    }
+  };
+
   let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
     uuid_idempotency_token: &request.uuid_idempotency_token,
-    job_type: InferenceJobType::Unknown,
+    job_type,
     inference_category: InferenceCategory::VoiceConversion,
     maybe_model_type: Some(model_inference_info.job_model_type),
     maybe_model_token: Some(&model_token),

crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/inference/enqueue_tts_request.rs

@@ -190,7 +190,7 @@ pub async fn enqueue_tts_request(
     // create the job record here!
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
         uuid_idempotency_token: &request.uuid_idempotency_token,
-        job_type: InferenceJobType::Unknown,
+        job_type: InferenceJobType::StyleTTS2,
         inference_category: InferenceCategory::TextToSpeech,
         maybe_model_type: Some(InferenceModelType::StyleTTS2), // NB: Model is static during inference
         maybe_model_token: None, // NB: Model is static during inference

crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voices/create_voice.rs

@@ -177,7 +177,7 @@ pub async fn create_voice_handler(
     // create the job record here! explore the table insert the inference args in here as json! keep it short
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
         uuid_idempotency_token: &request.uuid_idempotency_token,
-        job_type: InferenceJobType::Unknown,
+        job_type: InferenceJobType::StyleTTS2,
         inference_category: InferenceCategory::TextToSpeech,
         maybe_model_type: Some(InferenceModelType::StyleTTS2), // NB: Model is static during inference
         maybe_model_token: None, // NB: Model is static during inference

crates/service/web/storyteller_web/src/http_server/endpoints/workflows/enqueue_comfy_ui.rs

@@ -190,7 +190,7 @@ pub async fn enqueue_comfy_ui_handler(
 
     let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
         uuid_idempotency_token: &request.uuid_idempotency_token,
-        job_type: InferenceJobType::Unknown,
+        job_type: InferenceJobType::ComfyUi,
         inference_category: InferenceCategory::Workflow,
         maybe_model_type: Some(InferenceModelType::ComfyUi), // NB: Model is static during inference
         maybe_model_token: None, // NB: Model is static during inference