From 021ca222aeb203e4d44fdd7e9df33a89a1bef48e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 8 Sep 2024 21:50:33 -0400 Subject: [PATCH] lipsync usage, better estimate for lipsync jobs --- .../media_files/lipsync_usage.sql | 20 +++++++++++++++++++ .../inference_job_product_category.rs | 3 ++- .../web/batch_get_inference_job_status.rs | 5 +++++ .../web/get_inference_job_status.rs | 4 ++++ .../generic_inference/web/job_status.rs | 2 ++ .../web/list_session_jobs.rs | 5 +++++ .../utils/estimates/estimate_job_progress.rs | 11 +++++++++- 7 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 _sql/actions_and_analytics/media_files/lipsync_usage.sql diff --git a/_sql/actions_and_analytics/media_files/lipsync_usage.sql b/_sql/actions_and_analytics/media_files/lipsync_usage.sql new file mode 100644 index 0000000000..d97847229d --- /dev/null +++ b/_sql/actions_and_analytics/media_files/lipsync_usage.sql @@ -0,0 +1,20 @@ + +-- Total usage +select + count(*) as use_count +from generic_inference_jobs as gij +where gij.product_category = 'vid_face_fusion' + and gij.created_at >= (CURDATE() - INTERVAL 5 DAY) + +-- Usage by user +select + u.token, + u.username, + count(*) as use_count +from generic_inference_jobs as gij +left outer join users as u +on u.token = gij.maybe_creator_user_token +where gij.product_category = 'vid_face_fusion' + and gij.created_at >= (CURDATE() - INTERVAL 5 DAY) +group by u.token, u.username +order by use_count desc diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs index 69a26eaab7..6a9ce02cd1 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_job_product_category.rs @@ -13,10 +13,11 @@ use strum::EnumIter; /// /// YOU CAN ADD NEW VALUES, BUT DO NOT CHANGE EXISTING VALUES WITHOUT A MIGRATION STRATEGY. #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, Default)] #[serde(rename_all = "snake_case")] pub enum InferenceJobProductCategory { /// Download: GptSoVits + #[default] DownloadGptSoVits, /// Lipsync: Face Fusion diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs index 44d38c035f..1ff91e9620 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs @@ -6,6 +6,7 @@ use sqlx::pool::PoolConnection; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory; use enums::common::job_status_plus::JobStatusPlus; use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; @@ -90,6 +91,7 @@ SELECT jobs.maybe_creator_anonymous_visitor_token as maybe_creator_anonymous_visitor_token, jobs.creator_ip_address, + jobs.product_category, jobs.inference_category as inference_category, jobs.maybe_model_type, jobs.maybe_model_token, @@ -237,6 +239,7 @@ fn raw_records_to_public_result(records: Vec) -> V maybe_first_started_at: record.maybe_first_started_at, maybe_frontend_failure_category: record.maybe_frontend_failure_category, request_details: RequestDetails { + product_category: record.product_category, inference_category: record.inference_category, maybe_model_type: record.maybe_model_type, maybe_model_token: record.maybe_model_token, @@ -271,6 +274,7 @@ struct RawGenericInferenceJobStatus { pub maybe_creator_anonymous_visitor_token: Option, pub creator_ip_address: String, + pub product_category: InferenceJobProductCategory, pub inference_category: InferenceCategory, pub maybe_model_type: Option, pub maybe_model_token: Option, @@ -327,6 +331,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus { maybe_creator_user_token: UserToken::try_from_mysql_row_nullable(row, "maybe_creator_user_token")?, maybe_creator_anonymous_visitor_token: AnonymousVisitorTrackingToken::try_from_mysql_row_nullable(row, "maybe_creator_anonymous_visitor_token")?, creator_ip_address: row.try_get("creator_ip_address")?, + product_category: row.try_get("product_category")?, inference_category: InferenceCategory::try_from_mysql_row(row, "inference_category")?, maybe_model_type: row.try_get("maybe_model_type")?, maybe_model_token: row.try_get("maybe_model_token")?, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index a13b5ab4c3..a2d2b3a9be 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -6,6 +6,7 @@ use sqlx::pool::PoolConnection; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory; use enums::common::job_status_plus::JobStatusPlus; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use errors::AnyhowResult; @@ -49,6 +50,7 @@ SELECT jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`, jobs.creator_ip_address, + jobs.product_category as `product_category: enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory`, jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`, jobs.maybe_model_type, jobs.maybe_model_token, @@ -213,6 +215,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI maybe_first_started_at: record.maybe_first_started_at, maybe_frontend_failure_category: record.maybe_frontend_failure_category, request_details: RequestDetails { + product_category: record.product_category, inference_category: record.inference_category, maybe_model_type: record.maybe_model_type, maybe_model_token: record.maybe_model_token, @@ -245,6 +248,7 @@ struct RawGenericInferenceJobStatus { pub maybe_creator_anonymous_visitor_token: Option, pub creator_ip_address: String, + pub product_category: InferenceJobProductCategory, pub inference_category: InferenceCategory, pub maybe_model_type: Option, pub maybe_model_token: Option, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs index ba4674666b..bb055063fa 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs @@ -2,6 +2,7 @@ use chrono::{DateTime, Utc}; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory; use enums::common::job_status_plus::JobStatusPlus; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; @@ -42,6 +43,7 @@ pub struct GenericInferenceJobStatus { /// (We may want to present it in the "pending" UI.) #[derive(Debug, Default, Serialize)] pub struct RequestDetails { + pub product_category: InferenceJobProductCategory, pub inference_category: InferenceCategory, pub maybe_model_type: Option, // TODO: Strongly type pub maybe_model_token: Option, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs index f389d697f7..2fda308096 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs @@ -8,6 +8,7 @@ use sqlx::pool::PoolConnection; use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory; use enums::common::job_status_plus::JobStatusPlus; use enums::traits::mysql_from_row::MySqlFromRow; use errors::AnyhowResult; @@ -136,6 +137,7 @@ SELECT jobs.maybe_creator_anonymous_visitor_token as maybe_creator_anonymous_visitor_token, jobs.creator_ip_address, + jobs.product_category, jobs.inference_category as inference_category, jobs.maybe_model_type, jobs.maybe_model_token, @@ -283,6 +285,7 @@ fn raw_records_to_public_result(records: Vec) -> V maybe_first_started_at: record.maybe_first_started_at, maybe_frontend_failure_category: record.maybe_frontend_failure_category, request_details: RequestDetails { + product_category: record.product_category, inference_category: record.inference_category, maybe_model_type: record.maybe_model_type, maybe_model_token: record.maybe_model_token, @@ -317,6 +320,7 @@ struct RawGenericInferenceJobStatus { pub maybe_creator_anonymous_visitor_token: Option, pub creator_ip_address: String, + pub product_category: InferenceJobProductCategory, pub inference_category: InferenceCategory, pub maybe_model_type: Option, pub maybe_model_token: Option, @@ -373,6 +377,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus { maybe_creator_user_token: UserToken::try_from_mysql_row_nullable(row, "maybe_creator_user_token")?, maybe_creator_anonymous_visitor_token: AnonymousVisitorTrackingToken::try_from_mysql_row_nullable(row, "maybe_creator_anonymous_visitor_token")?, creator_ip_address: row.try_get("creator_ip_address")?, + product_category: row.try_get("product_category")?, inference_category: InferenceCategory::try_from_mysql_row(row, "inference_category")?, maybe_model_type: row.try_get("maybe_model_type")?, maybe_model_token: row.try_get("maybe_model_token")?, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs index 79fc416715..80fdee2154 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimates/estimate_job_progress.rs @@ -1,6 +1,7 @@ use std::cmp::min; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_job_product_category::InferenceJobProductCategory; use enums::common::job_status_plus::JobStatusPlus; use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs; use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus; @@ -14,6 +15,7 @@ const TTS_JOB_AVERAGE_SECONDS : u64 = 7; const VC_JOB_AVERAGE_SECONDS : u64 = 90; const LIVE_PORTRAIT_JOB_AVERAGE_SECONDS : u64 = 90; +const VID_FACE_FUSION_AVERAGE_SECONDS : u64 = 30; pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option<&PolymorphicInferenceArgs>) -> u8 { match job.status { @@ -49,7 +51,7 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option // This might also take parameters: the job arguments, text length, // video length (currently unavailable - we'll need to put that in the // job metadata), etc. - let progress = match job.request_details.inference_category { + let mut progress = match job.request_details.inference_category { // TODO: Better estimates for each of these job types. InferenceCategory::LipsyncAnimation => percent(duration_seconds, LIPSYNC_JOB_AVERAGE_SECONDS), InferenceCategory::TextToSpeech => percent(duration_seconds, TTS_JOB_AVERAGE_SECONDS), @@ -68,6 +70,13 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus, maybe_args: Option InferenceCategory::DeprecatedField => 0, // TODO(bt,2024-07-16): Read job type instead. }; + match job.request_details.product_category { + InferenceJobProductCategory::VidFaceFusion => { + progress = percent(duration_seconds, VID_FACE_FUSION_AVERAGE_SECONDS); + } + _ => {}, // Intentional Fallthrough + } + // We shouldn't show 100% if the job isn't complete. min(progress, 95) }