(from other laptop) - inference args work, inference-job boilerplate

crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs

@@ -21,6 +21,10 @@ pub enum InferenceModelType {
   #[serde(rename = "rvc_v2")]
   RvcV2,
 
+  // NB: sad_talker does use user-supplied models, so there is no "model token"
+  #[serde(rename = "sad_talker")]
+  SadTalker,
+
   #[serde(rename = "so_vits_svc")]
   SoVitsSvc,
 
@@ -41,6 +45,7 @@ impl InferenceModelType {
   pub fn to_str(&self) -> &'static str {
     match self {
       Self::RvcV2 => "rvc_v2",
+      Self::SadTalker => "sad_talker",
       Self::SoVitsSvc => "so_vits_svc",
       Self::Tacotron2 => "tacotron2",
       Self::Vits => "vits",
@@ -50,6 +55,7 @@ impl InferenceModelType {
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
       "rvc_v2" => Ok(Self::RvcV2),
+      "sad_talker" => Ok(Self::SadTalker),
       "so_vits_svc" => Ok(Self::SoVitsSvc),
       "tacotron2" => Ok(Self::Tacotron2),
       "vits" => Ok(Self::Vits),
@@ -62,6 +68,7 @@ impl InferenceModelType {
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
       InferenceModelType::RvcV2,
+      InferenceModelType::SadTalker,
       InferenceModelType::SoVitsSvc,
       InferenceModelType::Tacotron2,
       InferenceModelType::Vits,
@@ -77,6 +84,7 @@ mod tests {
   #[test]
   fn test_serialization() {
     assert_serialization(InferenceModelType::RvcV2, "rvc_v2");
+    assert_serialization(InferenceModelType::SadTalker, "sad_talker");
     assert_serialization(InferenceModelType::SoVitsSvc, "so_vits_svc");
     assert_serialization(InferenceModelType::Tacotron2, "tacotron2");
     assert_serialization(InferenceModelType::Vits, "vits");
@@ -85,6 +93,7 @@ mod tests {
   #[test]
   fn to_str() {
     assert_eq!(InferenceModelType::RvcV2.to_str(), "rvc_v2");
+    assert_eq!(InferenceModelType::SadTalker.to_str(), "sad_talker");
     assert_eq!(InferenceModelType::SoVitsSvc.to_str(), "so_vits_svc");
     assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2");
     assert_eq!(InferenceModelType::Vits.to_str(), "vits");
@@ -93,6 +102,7 @@ mod tests {
   #[test]
   fn from_str() {
     assert_eq!(InferenceModelType::from_str("rvc_v2").unwrap(), InferenceModelType::RvcV2);
+    assert_eq!(InferenceModelType::from_str("sad_talker").unwrap(), InferenceModelType::SadTalker);
     assert_eq!(InferenceModelType::from_str("so_vits_svc").unwrap(), InferenceModelType::SoVitsSvc);
     assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2);
     assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits);
@@ -102,8 +112,9 @@ mod tests {
   fn all_variants() {
     // Static check
     let mut variants = InferenceModelType::all_variants();
-    assert_eq!(variants.len(), 4);
+    assert_eq!(variants.len(), 5);
     assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2));
+    assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker));
     assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc));
     assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2));
     assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits));

crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs

@@ -1,3 +1,4 @@
+use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use errors::AnyhowResult;
 
@@ -47,15 +48,8 @@ pub enum FundamentalFrequencyMethodForJob {
 pub enum PolymorphicInferenceArgs {
   /// Lipsync Animation (Short name to save  space when serializing)
   /// This is SadTalker, not Wav2Lip.
-  La {
-    #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    maybe_audio_media_upload_token: Option<String>,
+  La (LipsyncArgs),
 
-    #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    maybe_image_media_upload_token: Option<String>,
-  },
   /// Text to speech. (Short name to save space when serializing.)
   Tts {
     // No arguments yet.
@@ -86,6 +80,7 @@ pub enum PolymorphicInferenceArgs {
   },
 }
 
+
 impl GenericInferenceArgs {
 
   pub fn from_json(json: &str) -> AnyhowResult<Self> {
@@ -117,22 +112,23 @@ impl InferenceCategoryAbbreviated {
 
 #[cfg(test)]
 mod tests {
-  use crate::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+  use crate::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+  use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
 
   #[test]
   fn typical_lipsync_animation_args_serialize() {
     let args = GenericInferenceArgs {
       inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation),
-      args: Some(PolymorphicInferenceArgs::La {
-        maybe_audio_media_upload_token: Some("foo".to_string()),
-        maybe_image_media_upload_token: Some("bar".to_string()),
-      }),
+      args: Some(PolymorphicInferenceArgs::La(LipsyncArgs {
+        maybe_audio_source: Some(LipsyncAnimationAudioSource::U("foo".to_string())),
+        maybe_image_source: Some(LipsyncAnimationImageSource::F("bar".to_string())),
+      })),
     };
 
     let json = serde_json::ser::to_string(&args).unwrap();
 
     // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating.
-    assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":"foo","i":"bar"}}}"#.to_string());
+    assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":{"U":"foo"},"i":{"F":"bar"}}}}"#.to_string());
 
     // NB: Make sure we don't overflow the DB field capacity (TEXT column).
     assert!(json.len() < 1000);

crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs

@@ -0,0 +1,125 @@
+//! Arguments for lipsync inferences
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct LipsyncArgs {
+  #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub maybe_audio_source: Option<LipsyncAnimationAudioSource>,
+
+  #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub maybe_image_source: Option<LipsyncAnimationImageSource>,
+}
+
+/// Audio sources can be one of several:
+///  - F: media_files (todo)
+///  - U: media_uploads (legacy)
+///  - T: tts_results (legacy)
+///  - V: voice_conversion_results (legacy)
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum LipsyncAnimationAudioSource {
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media File Token (media_files table)
+  /// Serde cannot yet rename enum variants.
+  F(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media Upload Token (media_uploads table)
+  /// Serde cannot yet rename enum variants.
+  U(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// TTS Result Token (tts_results table)
+  /// Serde cannot yet rename enum variants.
+  T(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Voice Conversion Result Token (voice_conversion_results table)
+  /// Serde cannot yet rename enum variants.
+  V(String),
+}
+
+// TODO: Hydrate to a public enum with easier to read names.
+impl LipsyncAnimationAudioSource {
+  pub fn media_file_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::F(token.to_string())
+  }
+  pub fn media_upload_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::U(token.to_string())
+  }
+  pub fn tts_result_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::T(token.to_string())
+  }
+  pub fn voice_conversion_result_token(token: &str) -> Self {
+    LipsyncAnimationAudioSource::V(token.to_string())
+  }
+}
+
+/// Video sources can be one of several:
+///  - F: media_files (todo)
+///  - U: media_uploads (legacy)
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum LipsyncAnimationImageSource {
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media File Token (media_files table)
+  /// Serde cannot yet rename enum variants.
+  F(String),
+
+  // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes.
+  /// Media Upload Token (media_uploads table)
+  /// Serde cannot yet rename enum variants.
+  U(String),
+}
+
+impl LipsyncAnimationImageSource {
+  pub fn media_file_token(token: &str) -> Self {
+    LipsyncAnimationImageSource::F(token.to_string())
+  }
+  pub fn media_upload_token(token: &str) -> Self {
+    LipsyncAnimationImageSource::U(token.to_string())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
+
+  #[test]
+  fn test_media_file() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::media_file_token("audio_media_file")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_file_token("image_media_file")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"F":"audio_media_file"},"i":{"F":"image_media_file"}}"#.to_string());
+  }
+
+  #[test]
+  fn test_media_upload() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::media_upload_token("audio_media_upload")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"U":"audio_media_upload"},"i":{"U":"image_media_upload"}}"#.to_string());
+  }
+
+  #[test]
+  fn test_tts_result() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::tts_result_token("audio_tts_result")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"T":"audio_tts_result"},"i":{"U":"image_media_upload"}}"#.to_string());
+  }
+  #[test]
+  fn test_voice_conversion_result() {
+    let args = LipsyncArgs {
+      maybe_audio_source: Some(LipsyncAnimationAudioSource::voice_conversion_result_token("audio_voice_conversion_result")),
+      maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")),
+    };
+    let json = serde_json::ser::to_string(&args).unwrap();
+    assert_eq!(json, r#"{"a":{"V":"audio_voice_conversion_result"},"i":{"U":"image_media_upload"}}"#.to_string());
+  }
+}

crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs

@@ -0,0 +1,2 @@
+pub mod generic_inference_args;
+pub mod lipsync_payload;

crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs

@@ -13,7 +13,7 @@ use std::future::Future;
 use std::path::Path;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use tokens::jobs::inference::InferenceJobToken;
-use crate::payloads::generic_inference_args::GenericInferenceArgs;
+use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs;
 
 /// table: generic_inference_jobs
 #[derive(Debug)]
@@ -26,7 +26,7 @@ pub struct AvailableInferenceJob {
   // Inference class
   pub inference_category: InferenceCategory,
 
-  pub maybe_model_type: Option<String>, // TODO: Strongly type.
+  pub maybe_model_type: Option<InferenceModelType>,
   pub maybe_model_token: Option<String>,
 
   pub maybe_input_source_token: Option<String>,
@@ -115,7 +115,11 @@ pub async fn list_available_generic_inference_jobs(
               .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
           inference_category: InferenceCategory::from_str(&record.inference_category)
               .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
-          maybe_model_type: record.maybe_model_type,
+          maybe_model_type: record.maybe_model_type
+              .as_deref()
+              .map(|model_type| InferenceModelType::from_str(model_type))
+              .transpose()
+              .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
           maybe_model_token: record.maybe_model_token,
           maybe_input_source_token: record.maybe_input_source_token,
           maybe_input_source_token_type: record.maybe_input_source_token_type,

crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs

@@ -1,5 +1,5 @@
 use anyhow::anyhow;
-use crate::payloads::generic_inference_args::GenericInferenceArgs;
+use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;

crates/service/job/inference_job/src/job/job_loop/process_single_job.rs

@@ -14,6 +14,7 @@ use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_s
 use r2d2_redis::redis::Commands;
 use redis_common::redis_keys::RedisKeys;
 use std::time::Instant;
+use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
 
 pub async fn process_single_job(
   job_dependencies: &JobDependencies,
@@ -167,6 +168,9 @@ async fn do_process_single_job(
   let mut maybe_result_entity : Option<ResultEntity>;
 
   let job_success_result = match job.inference_category {
+    InferenceCategory::LipsyncAnimation => {
+      process_single_lipsync_job(job_dependencies, job).await?
+    }
     InferenceCategory::TextToSpeech => {
       process_single_tts_job(job_dependencies, job).await?
     }

crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs

@@ -0,0 +1,2 @@
+pub mod process_single_lipsync_job;
+pub mod sad_talker;

crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs

@@ -0,0 +1,36 @@
+use anyhow::anyhow;
+use crate::job::job_loop::job_success_result::JobSuccessResult;
+use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
+use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs;
+use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs;
+use crate::job::job_types::vc::{rvc_v2, so_vits_svc};
+use crate::job_dependencies::JobDependencies;
+use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
+use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
+use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference;
+use std::time::Duration;
+use log::{error, info};
+use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
+use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
+use tokens::files::media_upload::MediaUploadToken;
+
+pub async fn process_single_lipsync_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
+
+  let job_success_result = match job.maybe_model_type {
+    Some(InferenceModelType::SadTalker) => {
+      //rvc_v2::process_job::process_job(RvcV2ProcessJobArgs {
+      //  job_dependencies,
+      //  job,
+      //  vc_model: &vc_model,
+      //  media_upload_token: &media_upload_token,
+      //  media_upload: &media_upload,
+      //}).await?
+      return Err(ProcessSingleJobError::Other(anyhow!("not yet implemented")))
+    }
+    Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))),
+    None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))),
+  };
+
+  Ok(job_success_result)
+}
+

crates/service/job/inference_job/src/job/job_types/mod.rs

@@ -1,2 +1,3 @@
+pub mod lipsync;
 pub mod tts;
 pub mod vc;

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs

@@ -15,9 +15,9 @@ use filesys::file_size::file_size;
 use log::{error, info, warn};
 use media::decode_basic_audio_info::decode_basic_audio_file_info;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
-use mysql_queries::payloads::generic_inference_args::PolymorphicInferenceArgs;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::media_uploads::get_media_upload_for_inference::{get_media_upload_for_inference, MediaUploadRecordForInference};
+use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
 use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference;
 use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs};
 use std::time::Instant;
@@ -149,6 +149,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
   let auto_predict_f0 = maybe_args
       .map(|args| match args {
         PolymorphicInferenceArgs::Tts { .. } => None,
+        PolymorphicInferenceArgs::La(_) => None,
         PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0,
       })
       .flatten()
@@ -157,6 +158,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
   let maybe_transpose = maybe_args
       .map(|args| match args {
         PolymorphicInferenceArgs::Tts { .. } => None,
+        PolymorphicInferenceArgs::La(_) => None,
         PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose,
       })
       .flatten();
@@ -164,6 +166,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
   let maybe_override_f0_method = maybe_args
       .map(|args| match args {
         PolymorphicInferenceArgs::Tts { .. } => None,
+        PolymorphicInferenceArgs::La(_) => None,
         PolymorphicInferenceArgs::Vc { override_f0_method, .. } =>
           *override_f0_method,
       })

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs

@@ -1,18 +1,17 @@
 use anyhow::anyhow;
 use container_common::anyhow_result::AnyhowResult;
+use crate::job::job_loop::command_exit_status::CommandExitStatus;
 use filesys::path_to_string::path_to_string;
 use log::info;
-use mysql_queries::payloads::generic_inference_args::FundamentalFrequencyMethodForJob;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::FundamentalFrequencyMethodForJob;
 use once_cell::sync::Lazy;
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
-use std::fs::OpenOptions;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
-use subprocess::{ExitStatus, Popen, PopenConfig, Redirection};
+use subprocess::{Popen, PopenConfig};
 use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions};
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
 
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!

crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs

@@ -15,11 +15,14 @@ use enums::common::visibility::Visibility;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
 use log::{info, warn};
-use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job};
 use std::sync::Arc;
 use tokens::files::media_upload::MediaUploadToken;
 use tokens::jobs::inference::InferenceJobToken;
+use tokens::tokens::media_files::MediaFileToken;
+use tokens::tokens::voice_conversion_results::VoiceConversionResultToken;
 use tokens::users::user::UserToken;
 use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL;
 
@@ -35,9 +38,8 @@ const ROUTING_TAG_HEADER_NAME : &'static str = "routing-tag";
 pub struct EnqueueLipsyncAnimationRequest {
   uuid_idempotency_token: String,
 
-  // TODO: Eventually accept result tokens for audio.
-  source_image_media_upload_token: MediaUploadToken,
-  source_audio_media_upload_token: MediaUploadToken,
+  audio_source: AudioSource,
+  image_source: ImageSource,
 
   creator_set_visibility: Option<Visibility>,
   is_storyteller_demo: Option<bool>,
@@ -46,6 +48,22 @@ pub struct EnqueueLipsyncAnimationRequest {
   animate_full_body: Option<bool>,
 }
 
+/// Treated as an enum. Only one of these may be set.
+#[derive(Deserialize)]
+pub struct AudioSource {
+  maybe_media_file_token: Option<MediaFileToken>,
+  maybe_media_upload_token: Option<MediaUploadToken>,
+  maybe_tts_result_token: Option<String>,
+  maybe_voice_conversion_result_token: Option<VoiceConversionResultToken>,
+}
+
+/// Treated as an enum. Only one of these may be set.
+#[derive(Deserialize)]
+pub struct ImageSource {
+  maybe_media_file_token: Option<MediaFileToken>,
+  maybe_media_upload_token: Option<MediaUploadToken>,
+}
+
 #[derive(Serialize)]
 pub struct EnqueueLipsyncAnimationSuccessResponse {
   pub success: bool,
@@ -190,9 +208,31 @@ pub async fn enqueue_lipsync_animation_handler(
 
   // ==================== LOOK UP MODEL INFO ==================== //
 
-  // TODO(bt): CHECK DATABASE!
-  let audio_media_token = request.source_audio_media_upload_token.clone();
-  let image_media_token = request.source_image_media_upload_token.clone();
+  // TODO(bt): CHECK DATABASE FOR TOKENS!
+
+  let audio_source = {
+    if let Some(ref token) = request.audio_source.maybe_media_file_token {
+      LipsyncAnimationAudioSource::media_file_token(token.as_str())
+    } else if let Some(ref token) = request.audio_source.maybe_media_upload_token {
+      LipsyncAnimationAudioSource::media_upload_token(token.as_str())
+    } else if let Some(ref token) = request.audio_source.maybe_tts_result_token {
+      LipsyncAnimationAudioSource::tts_result_token(token)
+    } else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token {
+      LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str())
+    } else {
+      return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string()));
+    }
+  };
+
+  let image_source = {
+    if let Some(ref token) = request.image_source.maybe_media_file_token {
+      LipsyncAnimationImageSource::media_file_token(token.as_str())
+    } else if let Some(ref token) = request.image_source.maybe_media_upload_token {
+      LipsyncAnimationImageSource::media_upload_token(token.as_str())
+    } else {
+      return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string()));
+    }
+  };
 
   // ==================== CHECK AND ENQUEUE ANIMATION ==================== //
 
@@ -224,28 +264,11 @@ pub async fn enqueue_lipsync_animation_handler(
 
   info!("Creating lipsync animation job record...");
 
-  // TODO: Cleanup.
-  let maybe_args = Some(PolymorphicInferenceArgs::La {
-    maybe_audio_media_upload_token: Some(audio_media_token.to_string()),
-    maybe_image_media_upload_token: Some(image_media_token.to_string()),
-  });
-
-  // TODO: SadTalker arguments.
-  //let set_args = request.auto_predict_f0.is_some()
-  //    || request.transpose.is_some()
-  //    || request.override_f0_method.is_some();
-
-  //if set_args {
-  //  maybe_args = Some(PolymorphicInferenceArgs::Vc {
-  //    auto_predict_f0: request.auto_predict_f0,
-  //    override_f0_method: request.override_f0_method.map(|f| match f {
-  //      FundamentalFrequencyMethod::Crepe => FundamentalFrequencyMethodForJob::Crepe,
-  //      FundamentalFrequencyMethod::Dio => FundamentalFrequencyMethodForJob::Dio,
-  //      FundamentalFrequencyMethod::Harvest => FundamentalFrequencyMethodForJob::Harvest,
-  //    }),
-  //    transpose: request.transpose,
-  //  });
-  //}
+  // TODO: other SadTalker arguments.
+  let maybe_args = Some(PolymorphicInferenceArgs::La(LipsyncArgs {
+    maybe_audio_source: Some(audio_source),
+    maybe_image_source: Some(image_source),
+  }));
 
   let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
     uuid_idempotency_token: &request.uuid_idempotency_token,

crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs

@@ -20,7 +20,7 @@ use http_server_common::request::get_request_api_token::get_request_api_token;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
 use log::{info, warn};
-use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated};
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
 use mysql_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder;
 use mysql_queries::queries::tts::tts_models::get_tts_model::TtsModelRecord;

crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs

@@ -19,7 +19,7 @@ use enums::common::visibility::Visibility;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
 use log::{error, info, warn};
-use mysql_queries::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job};
 use mysql_queries::queries::voice_conversion::model_info_lite::get_voice_conversion_model_info_lite::get_voice_conversion_model_info_lite_with_connection;
 use r2d2_redis::redis::Commands;