new enum to begin field deprecation

crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs

@@ -16,6 +16,11 @@ use utoipa::ToSchema;
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
 #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default, ToSchema)]
 pub enum InferenceCategory {
+  /// Deprecate this field !!!
+  /// We should drain all jobs from using this database field, then remove it.
+  #[serde(rename = "deprecated_field")]
+  DeprecatedField,
+
   /// Facial lipsync animation (eg. SadTalker, Wav2Lip)
   #[serde(rename = "lipsync_animation")]
   #[default]
@@ -68,6 +73,7 @@ impl_mysql_from_row!(InferenceCategory);
 impl InferenceCategory {
   pub fn to_str(&self) -> &'static str {
     match self {
+      Self::DeprecatedField => "deprecated_field",
       Self::LipsyncAnimation => "lipsync_animation",
       Self::TextToSpeech => "text_to_speech",
       Self::VoiceConversion => "voice_conversion",
@@ -82,6 +88,7 @@ impl InferenceCategory {
 
   pub fn from_str(value: &str) -> Result<Self, String> {
     match value {
+      "deprecated_field" => Ok(Self::DeprecatedField),
       "lipsync_animation" => Ok(Self::LipsyncAnimation),
       "text_to_speech" => Ok(Self::TextToSpeech),
       "voice_conversion" => Ok(Self::VoiceConversion),
@@ -99,6 +106,7 @@ impl InferenceCategory {
     // NB: BTreeSet is sorted
     // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
     BTreeSet::from([
+      Self::DeprecatedField,
       Self::LipsyncAnimation,
       Self::TextToSpeech,
       Self::VoiceConversion,
@@ -122,6 +130,7 @@ mod tests {
 
     #[test]
     fn test_serialization() {
+      assert_serialization(InferenceCategory::DeprecatedField, "deprecated_field");
       assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation");
       assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech");
       assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion");
@@ -135,6 +144,7 @@ mod tests {
 
     #[test]
     fn to_str() {
+      assert_eq!(InferenceCategory::DeprecatedField.to_str(), "deprecated_field");
       assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation");
       assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech");
       assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion");
@@ -148,6 +158,7 @@ mod tests {
 
     #[test]
     fn from_str() {
+      assert_eq!(InferenceCategory::from_str("deprecated_field").unwrap(), InferenceCategory::DeprecatedField);
       assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation);
       assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech);
       assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion);
@@ -163,7 +174,8 @@ mod tests {
     fn all_variants() {
       // Static check
       let mut variants = InferenceCategory::all_variants();
-      assert_eq!(variants.len(), 9);
+      assert_eq!(variants.len(), 10);
+      assert_eq!(variants.pop_first(), Some(InferenceCategory::DeprecatedField));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech));
       assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion));

crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs

@@ -148,36 +148,37 @@ impl GenericInferenceArgs {
   }
 }
 
-impl InferenceCategoryAbbreviated {
-  pub fn from_inference_category(category: InferenceCategory) -> Self {
-    match category {
-      InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation,
-      InferenceCategory::TextToSpeech => Self::TextToSpeech,
-      InferenceCategory::VoiceConversion => Self::VoiceConversion,
-      InferenceCategory::VideoFilter => Self::VideoFilter,
-      InferenceCategory::ImageGeneration => Self::ImageGeneration,
-      InferenceCategory::Mocap => Self::Mocap,
-      InferenceCategory::Workflow => Self::Workflow,
-      InferenceCategory::FormatConversion => Self::FormatConversion,
-      InferenceCategory::ConvertBvhToWorkflow => Self::ConvertBvhToWorkflow,
-    }
-  }
-
-  pub fn to_inference_category(self) -> InferenceCategory {
-    match self {
-      Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation,
-      Self::TextToSpeech => InferenceCategory::TextToSpeech,
-      Self::VoiceConversion => InferenceCategory::VoiceConversion,
-      Self::VideoFilter => InferenceCategory::VideoFilter,
-      Self::ImageGeneration =>InferenceCategory::ImageGeneration,
-      Self::Mocap => InferenceCategory::Mocap,
-      Self::Workflow => InferenceCategory::Workflow,
-      Self::LivePortrait => InferenceCategory::Workflow, // TODO(bt,2024-07-16): Not bidirectional/reflective
-      Self::FormatConversion => InferenceCategory::FormatConversion,
-      Self::ConvertBvhToWorkflow => InferenceCategory::ConvertBvhToWorkflow,
-    }
-  }
-}
+//impl InferenceCategoryAbbreviated {
+//  pub fn from_inference_category(category: InferenceCategory) -> Self {
+//    match category {
+//      InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation,
+//      InferenceCategory::TextToSpeech => Self::TextToSpeech,
+//      InferenceCategory::VoiceConversion => Self::VoiceConversion,
+//      InferenceCategory::VideoFilter => Self::VideoFilter,
+//      InferenceCategory::ImageGeneration => Self::ImageGeneration,
+//      InferenceCategory::Mocap => Self::Mocap,
+//      InferenceCategory::Workflow => Self::Workflow,
+//      InferenceCategory::FormatConversion => Self::FormatConversion,
+//      InferenceCategory::ConvertBvhToWorkflow => Self::ConvertBvhToWorkflow,
+//      InferenceCategory::DeprecatedField => Self::LivePortrait, // TODO(bt,2024-07-16): This isn't correct
+//    }
+//  }
+//
+//  pub fn to_inference_category(self) -> InferenceCategory {
+//    match self {
+//      Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation,
+//      Self::TextToSpeech => InferenceCategory::TextToSpeech,
+//      Self::VoiceConversion => InferenceCategory::VoiceConversion,
+//      Self::VideoFilter => InferenceCategory::VideoFilter,
+//      Self::ImageGeneration =>InferenceCategory::ImageGeneration,
+//      Self::Mocap => InferenceCategory::Mocap,
+//      Self::Workflow => InferenceCategory::Workflow,
+//      Self::LivePortrait => InferenceCategory::DeprecatedField, // TODO(bt,2024-07-16): This isn't correct
+//      Self::FormatConversion => InferenceCategory::FormatConversion,
+//      Self::ConvertBvhToWorkflow => InferenceCategory::ConvertBvhToWorkflow,
+//    }
+//  }
+//}
 
 #[cfg(test)]
 mod tests {

crates/lib/mysql_queries/src/payloads/generic_inference_args/live_portrait_payload.rs

@@ -17,10 +17,6 @@ pub struct LivePortraitPayload {
   #[serde(skip_serializing_if = "Option::is_none")]
   pub remove_watermark: Option<bool>,
 
-  #[serde(rename = "cv")]
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub creator_visibility: Option<Visibility>,
-
   /// This is a debugging flag.
   #[serde(rename = "sp")]
   #[serde(skip_serializing_if = "Option::is_none")]

crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs

@@ -28,6 +28,7 @@ pub struct AvailableInferenceJob {
   pub uuid_idempotency_token: String, // TODO: This is temporarily being used for upload paths.
 
   // Inference class
+  pub job_type: InferenceJobType,
   pub inference_category: InferenceCategory,
 
   pub maybe_model_type: Option<InferenceModelType>,
@@ -124,6 +125,8 @@ pub async fn list_available_generic_inference_jobs(
               .map(|s| MediaFileToken::new_from_str(&s)),
           creator_set_visibility: Visibility::from_str(&record.creator_set_visibility)
               .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
+          job_type: InferenceJobType::from_str(&record.job_type)
+              .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
           inference_category: InferenceCategory::from_str(&record.inference_category)
               .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix.
           maybe_model_type: record.maybe_model_type
@@ -211,6 +214,7 @@ SELECT
   token as inference_job_token,
   uuid_idempotency_token,
 
+  job_type,
   inference_category,
   maybe_model_type,
   maybe_model_token,
@@ -308,6 +312,7 @@ struct AvailableInferenceJobRawInternal {
 
   // Inference information
   //pub inference_category: InferenceCategory,
+  pub job_type: String,
   pub inference_category: String,
 
   pub maybe_model_type: Option<String>,

crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs

@@ -176,6 +176,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
             InferenceCategory::Workflow => Some("Workflow"),
             InferenceCategory::FormatConversion => Some("format conversion"),
             InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
+            InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix
           };
         }
 
@@ -191,6 +192,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
           InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::ConvertBvhToWorkflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
+          InferenceCategory::DeprecatedField => (true, None), // TODO(bt,2024-07-16): We'll need to read another field!
         };
 
         // NB: We've moved voice conversion out of their own table and into media_files

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -152,6 +152,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
       InferenceCategory::Workflow => Some("Workflow"),
       InferenceCategory::FormatConversion => Some("format conversion"),
       InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
+      InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix
     };
   }
 
@@ -167,6 +168,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
     InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
     InferenceCategory::ConvertBvhToWorkflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
+    InferenceCategory::DeprecatedField => (true, None), // TODO(bt,2024-07-16): We'll need to read another field!
   };
 
   // NB: We've moved voice conversion out of their own table and into media_files

crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs

@@ -222,6 +222,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
             InferenceCategory::Workflow => Some("Workflow"),
             InferenceCategory::FormatConversion => Some("format conversion"),
             InferenceCategory::ConvertBvhToWorkflow => Some("BVH to Workflow"),
+            InferenceCategory::DeprecatedField => Some("Job"), // TODO(bt,2024-07-16): Fix
           };
         }
 
@@ -237,6 +238,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
           InferenceCategory::Workflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::FormatConversion => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
           InferenceCategory::ConvertBvhToWorkflow => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
+          InferenceCategory::DeprecatedField => (true, None), // TODO(bt,2024-07-16): We'll need to read another field!
         };
 
         // NB: We've moved voice conversion out of their own table and into media_files

crates/service/job/inference_job/src/job/job_loop/process_single_job.rs

@@ -6,6 +6,7 @@ use opentelemetry::KeyValue as OtelAttribute;
 use r2d2_redis::redis::Commands;
 
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_pending_and_grab_lock::mark_generic_inference_job_pending_and_grab_lock;
 use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_successfully_done::mark_generic_inference_job_successfully_done;
@@ -181,6 +182,8 @@ async fn do_process_single_job(
 
   // ==================== HANDLE DIFFERENT INFERENCE TYPES ==================== //
 
+  // TODO(bt,2024-07-16): This logic needs to be cleaned up and better coordinated with the setup env vars.
+
   let job_success_result = match job.inference_category {
     InferenceCategory::LipsyncAnimation => {
       process_single_lipsync_job(job_dependencies, job).await?
@@ -200,15 +203,29 @@ async fn do_process_single_job(
     InferenceCategory::Mocap => {
       process_single_mc_job(job_dependencies, job).await?
     }
-    InferenceCategory::Workflow => {
-      process_single_workflow_job(job_dependencies, job).await?
-    }
     InferenceCategory::FormatConversion => {
       process_single_format_conversion_job(job_dependencies, job).await?
     }
     InferenceCategory::ConvertBvhToWorkflow => {
       process_single_render_engine_scene_job(job_dependencies, job).await?
     }
+    InferenceCategory::Workflow => {
+      process_single_workflow_job(job_dependencies, job).await?
+    }
+    InferenceCategory::DeprecatedField => {
+      match job.job_type {
+        InferenceJobType::VideoRender
+        | InferenceJobType::LivePortrait
+        | InferenceJobType::ComfyUi => {
+          // NB: These are all comfy workflow jobs too
+          process_single_workflow_job(job_dependencies, job).await?
+        }
+        _ => {
+          return Err(ProcessSingleJobError::InvalidJob(
+            anyhow!("invalid job for dispatch. type: {:?}", job.job_type)))
+        }
+      }
+    }
   };
 
   let maybe_entity_type = job_success_result.maybe_result_entity

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/batch_get_inference_job_status_handler.rs

@@ -358,8 +358,11 @@ fn db_record_to_response_payload(
         InferenceCategory::ConvertBvhToWorkflow => {
           "".to_string()
         }
+        InferenceCategory::DeprecatedField => {
+          "".to_string() // TODO(bt,2024-07-16): Read job type instead
+        }
       };
-     
+
       BatchResultDetailsResponse {
         entity_type: result_details.entity_type,
         entity_token: result_details.entity_token,

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status_handler.rs

@@ -348,6 +348,9 @@ fn record_to_payload(
         InferenceCategory::ConvertBvhToWorkflow => {
           "".to_string()
         }
+        InferenceCategory::DeprecatedField => {
+          "".to_string() // TODO(bt,2024-07-15): Fix. This should read job type.
+        }
       };
      
       ResultDetailsResponse {

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/list_session_jobs_handler.rs

@@ -414,6 +414,9 @@ fn db_record_to_response_payload(
         InferenceCategory::ConvertBvhToWorkflow => {
           "".to_string()
         }
+        InferenceCategory::DeprecatedField => {
+          "".to_string() // TODO(bt,2024-07-16): Read job type instead
+        }
       };
      
       ListSessionResultDetailsResponse {

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimate_job_progress.rs

@@ -61,6 +61,7 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus) -> u8 {
     InferenceCategory::ImageGeneration => 0,
     InferenceCategory::VideoFilter => 0,
     InferenceCategory::ConvertBvhToWorkflow => 0,
+    InferenceCategory::DeprecatedField => 0, // TODO(bt,2024-07-16): Read job type instead.
   };
 
   // We shouldn't show 100% if the job isn't complete.

crates/service/web/storyteller_web/src/http_server/endpoints/workflows/enqueue/enqueue_live_portrait_workflow_handler.rs

@@ -19,7 +19,8 @@ use enums::common::visibility::Visibility;
 use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
-use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::live_portrait_payload::LivePortraitPayload;
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
 use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
 use primitives::traits::trim_or_emptyable::TrimOrEmptyable;
@@ -36,6 +37,7 @@ use crate::http_server::web_utils::require_user_session::RequireUserSessionError
 use crate::http_server::web_utils::require_user_session_using_connection::require_user_session_using_connection;
 use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
 use crate::state::server_state::ServerState;
+use crate::util::cleaners::empty_media_file_token_to_null::empty_media_file_token_to_null;
 
 #[derive(Deserialize, ToSchema)]
 pub struct EnqueueLivePortraitWorkflowRequest {
@@ -193,63 +195,19 @@ pub async fn enqueue_live_portrait_workflow_handler(
 
   let _is_allowed_expensive_generation = is_staff || has_paid_plan;
 
-//  let inference_args = WorkflowArgs {
-//    // Main text prompts
-//    positive_prompt: coordinated_args.prompt.new_string_trim_or_empty(),
-//    negative_prompt: request.negative_prompt.new_string_trim_or_empty(),
-//    travel_prompt: coordinated_args.travel_prompt.new_string_trim_or_empty(),
-//
-//    // Input files
-//    maybe_input_file: Some(request.input_file.clone()),
-//    maybe_input_depth_file: empty_token_to_null(request.input_depth_file.as_ref()),
-//    maybe_input_normal_file: empty_token_to_null(request.input_normal_file.as_ref()),
-//    maybe_input_outline_file: empty_token_to_null(request.input_outline_file.as_ref()),
-//    global_ip_adapter_token: empty_token_to_null(request.global_ipa_media_token.as_ref()),
-//
-//    // Other inputs
-//    style_name: Some(request.style),
-//    creator_visibility: Some(set_visibility),
-//    trim_start_milliseconds: Some(trim_start_millis),
-//    trim_end_milliseconds: Some(trim_end_millis),
-//    strength: maybe_strength,
-//    frame_skip: request.frame_skip,
-//
-//    // Flags
-//    disable_lcm: coordinated_args.disable_lcm,
-//    use_cinematic: coordinated_args.use_cinematic,
-//    use_face_detailer: coordinated_args.use_face_detailer,
-//    use_upscaler: coordinated_args.use_upscaler,
-//    remove_watermark: coordinated_args.remove_watermark,
-//    lipsync_enabled: coordinated_args.use_lipsync,
-//    enable_lipsync: coordinated_args.use_lipsync, // TODO(bt): We can stop writing this flag after we re-deploy the job.
-//
-//    // TODO: Get rid of the temporary flags.
-//    rollout_python_workflow_args: None,
-//    skip_process_video: get_request_header_optional(&http_request, "SKIP-PROCESS-VIDEO")
-//        .map(|value| str_to_bool(&value)),
-//    sleep_millis: get_request_header_optional(&http_request, "SLEEP-MILLIS")
-//        .and_then(|value| try_str_to_num(&value).ok()),
-//
-//    // The new, simplified enqueuing doesn't care about the following parameters:
-//    maybe_lora_model: None,
-//    maybe_json_modifications: None,
-//    maybe_workflow_config: None,
-//    maybe_output_path: None,
-//    maybe_google_drive_link: None,
-//    maybe_title: None,
-//    maybe_commit_hash:None,
-//    maybe_description:None,
-//    trim_start_seconds: None,
-//    trim_end_seconds: None,
-//    target_fps: None,
-//  };
+  let payload = LivePortraitPayload {
+    portrait_media_file_token: empty_media_file_token_to_null(Some(&request.portrait_media_file_token)),
+    driver_media_file_token: empty_media_file_token_to_null(Some(&request.driver_media_file_token)),
+    remove_watermark: None,
+    sleep_millis: None,
+  };
 
   info!("Creating ComfyUI job record...");
 
   let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
     uuid_idempotency_token: &request.uuid_idempotency_token,
     job_type: InferenceJobType::LivePortrait,
-    inference_category: InferenceCategory::Workflow,
+    inference_category: InferenceCategory::DeprecatedField,
     maybe_model_type: None,
     maybe_model_token: None,
     maybe_input_source_token: None,
@@ -259,14 +217,13 @@ pub async fn enqueue_live_portrait_workflow_handler(
     maybe_raw_inference_text: None,
     maybe_max_duration_seconds: None,
     maybe_inference_args: Some(GenericInferenceArgs {
-      inference_category: Some(InferenceCategoryAbbreviated::Workflow),
-      //args: Some(PolymorphicInferenceArgs::Cu(inference_args)), // TODO
-      args: None,
+      inference_category: Some(InferenceCategoryAbbreviated::LivePortrait),
+      args: Some(PolymorphicInferenceArgs::Lp(payload)),
     }),
     maybe_creator_user_token: Some(&user_session.user_token_typed),
     maybe_avt_token: maybe_avt_token.as_ref(),
     creator_ip_address: &ip_address,
-    creator_set_visibility:  set_visibility,
+    creator_set_visibility: set_visibility,
     priority_level,
     requires_keepalive: plan.workflow_requires_frontend_keepalive(),
     is_debug_request,