lipsync usage, better estimate for lipsync jobs

_sql/actions_and_analytics/media_files/lipsync_usage.sql

@@ -0,0 +1,20 @@
+
+-- Total usage
+select
+  count(*) as use_count
+from generic_inference_jobs as gij
+where gij.product_category = 'vid_face_fusion'
+  and gij.created_at >= (CURDATE() - INTERVAL 5 DAY)
+
+-- Usage by user
+select
+  u.token,
+  u.username,
+  count(*) as use_count
+from generic_inference_jobs as gij
+left outer join users as u
+on u.token = gij.maybe_creator_user_token
+where gij.product_category = 'vid_face_fusion'
+  and gij.created_at >= (CURDATE() - INTERVAL 5 DAY)
+group by u.token, u.username
+order by use_count desc

crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs

@@ -13,10 +13,11 @@ use strum::EnumIter;
 ///
 /// YOU CAN ADD NEW VALUES, BUT DO NOT CHANGE EXISTING VALUES WITHOUT A MIGRATION STRATEGY.
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
-#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
+#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)]
 #[serde(rename_all = "snake_case")]
 pub enum InferenceJobProductCategory {
   /// Download: GptSoVits
+  #[default]
   DownloadGptSoVits,
 
   /// Lipsync: Face Fusion

crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs

@@ -6,6 +6,7 @@ use sqlx::pool::PoolConnection;
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
 use enums::common::job_status_plus::JobStatusPlus;
 use enums::traits::mysql_from_row::MySqlFromRow;
 use errors::AnyhowResult;
@@ -90,6 +91,7 @@ SELECT
     jobs.maybe_creator_anonymous_visitor_token as maybe_creator_anonymous_visitor_token,
     jobs.creator_ip_address,
 
+    jobs.product_category,
     jobs.inference_category as inference_category,
     jobs.maybe_model_type,
     jobs.maybe_model_token,
@@ -237,6 +239,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
           maybe_first_started_at: record.maybe_first_started_at,
           maybe_frontend_failure_category: record.maybe_frontend_failure_category,
           request_details: RequestDetails {
+            product_category: record.product_category,
             inference_category: record.inference_category,
             maybe_model_type: record.maybe_model_type,
             maybe_model_token: record.maybe_model_token,
@@ -271,6 +274,7 @@ struct RawGenericInferenceJobStatus {
   pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
   pub creator_ip_address: String,
 
+  pub product_category: InferenceJobProductCategory,
   pub inference_category: InferenceCategory,
   pub maybe_model_type: Option<String>,
   pub maybe_model_token: Option<String>,
@@ -327,6 +331,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus {
       maybe_creator_user_token: UserToken::try_from_mysql_row_nullable(row, "maybe_creator_user_token")?,
       maybe_creator_anonymous_visitor_token: AnonymousVisitorTrackingToken::try_from_mysql_row_nullable(row, "maybe_creator_anonymous_visitor_token")?,
       creator_ip_address: row.try_get("creator_ip_address")?,
+      product_category: row.try_get("product_category")?,
       inference_category: InferenceCategory::try_from_mysql_row(row, "inference_category")?,
       maybe_model_type: row.try_get("maybe_model_type")?,
       maybe_model_token: row.try_get("maybe_model_token")?,

crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs

@@ -6,6 +6,7 @@ use sqlx::pool::PoolConnection;
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
 use enums::common::job_status_plus::JobStatusPlus;
 use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
 use errors::AnyhowResult;
@@ -49,6 +50,7 @@ SELECT
     jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,
     jobs.creator_ip_address,
 
+    jobs.product_category as `product_category: enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory`,
     jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,
     jobs.maybe_model_type,
     jobs.maybe_model_token,
@@ -213,6 +215,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
     maybe_first_started_at: record.maybe_first_started_at,
     maybe_frontend_failure_category: record.maybe_frontend_failure_category,
     request_details: RequestDetails {
+      product_category: record.product_category,
       inference_category: record.inference_category,
       maybe_model_type: record.maybe_model_type,
       maybe_model_token: record.maybe_model_token,
@@ -245,6 +248,7 @@ struct RawGenericInferenceJobStatus {
   pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
   pub creator_ip_address: String,
 
+  pub product_category: InferenceJobProductCategory,
   pub inference_category: InferenceCategory,
   pub maybe_model_type: Option<String>,
   pub maybe_model_token: Option<String>,

crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs

@@ -2,6 +2,7 @@ use chrono::{DateTime, Utc};
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
 use enums::common::job_status_plus::JobStatusPlus;
 use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
 use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
@@ -42,6 +43,7 @@ pub struct GenericInferenceJobStatus {
 /// (We may want to present it in the "pending" UI.)
 #[derive(Debug, Default, Serialize)]
 pub struct RequestDetails {
+  pub product_category: InferenceJobProductCategory,
   pub inference_category: InferenceCategory,
   pub maybe_model_type: Option<String>, // TODO: Strongly type
   pub maybe_model_token: Option<String>,

crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs

@@ -8,6 +8,7 @@ use sqlx::pool::PoolConnection;
 
 use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
 use enums::common::job_status_plus::JobStatusPlus;
 use enums::traits::mysql_from_row::MySqlFromRow;
 use errors::AnyhowResult;
@@ -136,6 +137,7 @@ SELECT
     jobs.maybe_creator_anonymous_visitor_token as maybe_creator_anonymous_visitor_token,
     jobs.creator_ip_address,
 
+    jobs.product_category,
     jobs.inference_category as inference_category,
     jobs.maybe_model_type,
     jobs.maybe_model_token,
@@ -283,6 +285,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
           maybe_first_started_at: record.maybe_first_started_at,
           maybe_frontend_failure_category: record.maybe_frontend_failure_category,
           request_details: RequestDetails {
+            product_category: record.product_category,
             inference_category: record.inference_category,
             maybe_model_type: record.maybe_model_type,
             maybe_model_token: record.maybe_model_token,
@@ -317,6 +320,7 @@ struct RawGenericInferenceJobStatus {
   pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
   pub creator_ip_address: String,
 
+  pub product_category: InferenceJobProductCategory,
   pub inference_category: InferenceCategory,
   pub maybe_model_type: Option<String>,
   pub maybe_model_token: Option<String>,
@@ -373,6 +377,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus {
       maybe_creator_user_token: UserToken::try_from_mysql_row_nullable(row, "maybe_creator_user_token")?,
       maybe_creator_anonymous_visitor_token: AnonymousVisitorTrackingToken::try_from_mysql_row_nullable(row, "maybe_creator_anonymous_visitor_token")?,
       creator_ip_address: row.try_get("creator_ip_address")?,
+      product_category: row.try_get("product_category")?,
       inference_category: InferenceCategory::try_from_mysql_row(row, "inference_category")?,
       maybe_model_type: row.try_get("maybe_model_type")?,
       maybe_model_token: row.try_get("maybe_model_token")?,

crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs

@@ -1,6 +1,7 @@
 use std::cmp::min;
 
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory;
 use enums::common::job_status_plus::JobStatusPlus;
 use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
 use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus;
@@ -14,6 +15,7 @@ const TTS_JOB_AVERAGE_SECONDS : u64 = 7;
 const VC_JOB_AVERAGE_SECONDS : u64 = 90;
 
 const LIVE_PORTRAIT_JOB_AVERAGE_SECONDS : u64 = 90;
+const VID_FACE_FUSION_AVERAGE_SECONDS : u64 = 30;
 
 pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option<&PolymorphicInferenceArgs>) -> u8 {
   match job.status {
@@ -49,7 +51,7 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option
   //  This might also take parameters: the job arguments, text length,
   //  video length (currently unavailable - we'll need to put that in the
   //  job metadata), etc.
-  let progress = match job.request_details.inference_category {
+  let mut progress = match job.request_details.inference_category {
     // TODO: Better estimates for each of these job types.
     InferenceCategory::LipsyncAnimation => percent(duration_seconds, LIPSYNC_JOB_AVERAGE_SECONDS),
     InferenceCategory::TextToSpeech => percent(duration_seconds, TTS_JOB_AVERAGE_SECONDS),
@@ -68,6 +70,13 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option
     InferenceCategory::DeprecatedField => 0, // TODO(bt,2024-07-16): Read job type instead.
   };
 
+  match job.request_details.product_category {
+    InferenceJobProductCategory::VidFaceFusion => {
+      progress = percent(duration_seconds, VID_FACE_FUSION_AVERAGE_SECONDS);
+    }
+    _ => {}, // Intentional Fallthrough
+  }
+
   // We shouldn't show 100% if the job isn't complete.
   min(progress, 95)
 }