From ee90f03436565a1256e79acfbcd1b2508e1c834a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 Apr 2023 05:11:57 -0700 Subject: [PATCH] add extra inference job stats --- crates/lib/mysql_queries/sqlx-data.json | 466 ++++++++++-------- .../web/get_inference_job_status.rs | 25 +- .../inference_job/get_inference_job_status.rs | 13 +- 3 files changed, 295 insertions(+), 209 deletions(-) diff --git a/crates/lib/mysql_queries/sqlx-data.json b/crates/lib/mysql_queries/sqlx-data.json index 29f7749cbd..229b1879e2 100644 --- a/crates/lib/mysql_queries/sqlx-data.json +++ b/crates/lib/mysql_queries/sqlx-data.json @@ -8682,6 +8682,265 @@ }, "query": "\nSELECT\n c.token as `token: tokens::tokens::comments::CommentToken`,\n\n c.entity_type as `entity_type: enums::by_table::comments::comment_entity_type::CommentEntityType`,\n c.entity_token,\n\n c.user_token as `user_token: tokens::users::user::UserToken`,\n u.username,\n u.display_name as user_display_name,\n u.email_gravatar_hash as user_gravatar_hash,\n\n c.comment_markdown,\n c.comment_rendered_html,\n\n c.creator_ip_address,\n c.editor_ip_address,\n\n c.created_at,\n c.updated_at,\n c.edited_at,\n c.user_deleted_at,\n c.mod_deleted_at,\n c.object_owner_deleted_at\n\nFROM\n comments AS c\nJOIN users AS u\n ON c.user_token = u.token\nWHERE\n c.token = ?\n " }, + "75243d89645654dbcca7ddeff925230508b0e1c887bdb9d2e45812e7538ed63d": { + "describe": { + "columns": [ + { + "name": "job_token: tokens::jobs::inference::InferenceJobToken", + "ordinal": 0, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4229 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "status", + "ordinal": 1, + "type_info": { + "char_set": 224, + "flags": { + "bits": 393 + }, + "max_size": 64, + "type": "String" + } + }, + { + "name": "attempt_count", + "ordinal": 2, + "type_info": { + "char_set": 63, + "flags": { + "bits": 33 + }, + "max_size": 5, + "type": "Short" + } + }, + { + "name": "inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory", + "ordinal": 3, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4233 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "maybe_model_type", + "ordinal": 4, + "type_info": { + "char_set": 224, + "flags": { + "bits": 136 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "maybe_model_token", + "ordinal": 5, + "type_info": { + "char_set": 224, + "flags": { + "bits": 136 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "maybe_raw_inference_text", + "ordinal": 6, + "type_info": { + "char_set": 224, + "flags": { + "bits": 144 + }, + "max_size": 262140, + "type": "Blob" + } + }, + { + "name": "maybe_result_entity_type", + "ordinal": 7, + "type_info": { + "char_set": 224, + "flags": { + "bits": 128 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "maybe_result_entity_token", + "ordinal": 8, + "type_info": { + "char_set": 224, + "flags": { + "bits": 136 + }, + "max_size": 128, + "type": "VarString" + } + }, + { + "name": "maybe_tts_model_title", + "ordinal": 9, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4224 + }, + "max_size": 1020, + "type": "VarString" + } + }, + { + "name": "maybe_voice_conversion_model_title", + "ordinal": 10, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4224 + }, + "max_size": 1020, + "type": "VarString" + } + }, + { + "name": "maybe_tts_public_bucket_path", + "ordinal": 11, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4224 + }, + "max_size": 1020, + "type": "VarString" + } + }, + { + "name": "maybe_voice_conversion_public_bucket_hash", + "ordinal": 12, + "type_info": { + "char_set": 224, + "flags": { + "bits": 4224 + }, + "max_size": 256, + "type": "VarString" + } + }, + { + "name": "maybe_assigned_worker", + "ordinal": 13, + "type_info": { + "char_set": 224, + "flags": { + "bits": 128 + }, + "max_size": 512, + "type": "VarString" + } + }, + { + "name": "maybe_assigned_cluster", + "ordinal": 14, + "type_info": { + "char_set": 224, + "flags": { + "bits": 128 + }, + "max_size": 512, + "type": "VarString" + } + }, + { + "name": "created_at", + "ordinal": 15, + "type_info": { + "char_set": 63, + "flags": { + "bits": 1153 + }, + "max_size": 19, + "type": "Timestamp" + } + }, + { + "name": "updated_at", + "ordinal": 16, + "type_info": { + "char_set": 63, + "flags": { + "bits": 9345 + }, + "max_size": 19, + "type": "Timestamp" + } + }, + { + "name": "maybe_first_started_at", + "ordinal": 17, + "type_info": { + "char_set": 63, + "flags": { + "bits": 128 + }, + "max_size": 19, + "type": "Timestamp" + } + }, + { + "name": "maybe_successfully_completed_at", + "ordinal": 18, + "type_info": { + "char_set": 63, + "flags": { + "bits": 128 + }, + "max_size": 19, + "type": "Timestamp" + } + } + ], + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true + ], + "parameters": { + "Right": 1 + } + }, + "query": "\nSELECT\n jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`,\n\n jobs.status,\n jobs.attempt_count,\n\n jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n jobs.maybe_model_type,\n jobs.maybe_model_token,\n jobs.maybe_raw_inference_text,\n\n jobs.on_success_result_entity_type as maybe_result_entity_type,\n jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n tts_models.title as maybe_tts_model_title,\n voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n jobs.assigned_worker as maybe_assigned_worker,\n jobs.assigned_cluster as maybe_assigned_cluster,\n\n jobs.created_at,\n jobs.updated_at,\n\n jobs.first_started_at as maybe_first_started_at,\n jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\n\nWHERE jobs.token = ?\n " + }, "7622292c05fe1b5832136762042780a12a02048166bb1b4492dfd627c48c8968": { "describe": { "columns": [ @@ -11934,213 +12193,6 @@ }, "query": "\nINSERT INTO voice_conversion_models\nSET\n token = ?,\n model_type = ?,\n title = ?,\n description_markdown = '',\n description_rendered_html = '',\n creator_user_token = ?,\n ip_address_creation = ?,\n ip_address_last_update = ?,\n original_download_url = ?,\n private_bucket_hash = ?,\n private_bucket_object_name = ?,\n file_size_bytes = ?\n " }, - "a90c614a2add86a771bbe9626141b5a90beffbbf0c73cab2d45388f49a747bc3": { - "describe": { - "columns": [ - { - "name": "job_token: tokens::jobs::inference::InferenceJobToken", - "ordinal": 0, - "type_info": { - "char_set": 224, - "flags": { - "bits": 4229 - }, - "max_size": 128, - "type": "VarString" - } - }, - { - "name": "status", - "ordinal": 1, - "type_info": { - "char_set": 224, - "flags": { - "bits": 393 - }, - "max_size": 64, - "type": "String" - } - }, - { - "name": "attempt_count", - "ordinal": 2, - "type_info": { - "char_set": 63, - "flags": { - "bits": 33 - }, - "max_size": 5, - "type": "Short" - } - }, - { - "name": "inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory", - "ordinal": 3, - "type_info": { - "char_set": 224, - "flags": { - "bits": 4233 - }, - "max_size": 128, - "type": "VarString" - } - }, - { - "name": "maybe_model_type", - "ordinal": 4, - "type_info": { - "char_set": 224, - "flags": { - "bits": 136 - }, - "max_size": 128, - "type": "VarString" - } - }, - { - "name": "maybe_model_token", - "ordinal": 5, - "type_info": { - "char_set": 224, - "flags": { - "bits": 136 - }, - "max_size": 128, - "type": "VarString" - } - }, - { - "name": "maybe_raw_inference_text", - "ordinal": 6, - "type_info": { - "char_set": 224, - "flags": { - "bits": 144 - }, - "max_size": 262140, - "type": "Blob" - } - }, - { - "name": "maybe_result_entity_type", - "ordinal": 7, - "type_info": { - "char_set": 224, - "flags": { - "bits": 128 - }, - "max_size": 128, - "type": "VarString" - } - }, - { - "name": "maybe_result_entity_token", - "ordinal": 8, - "type_info": { - "char_set": 224, - "flags": { - "bits": 136 - }, - "max_size": 128, - "type": "VarString" - } - }, - { - "name": "maybe_tts_model_title", - "ordinal": 9, - "type_info": { - "char_set": 224, - "flags": { - "bits": 4224 - }, - "max_size": 1020, - "type": "VarString" - } - }, - { - "name": "maybe_voice_conversion_model_title", - "ordinal": 10, - "type_info": { - "char_set": 224, - "flags": { - "bits": 4224 - }, - "max_size": 1020, - "type": "VarString" - } - }, - { - "name": "maybe_tts_public_bucket_path", - "ordinal": 11, - "type_info": { - "char_set": 224, - "flags": { - "bits": 4224 - }, - "max_size": 1020, - "type": "VarString" - } - }, - { - "name": "maybe_voice_conversion_public_bucket_hash", - "ordinal": 12, - "type_info": { - "char_set": 224, - "flags": { - "bits": 4224 - }, - "max_size": 256, - "type": "VarString" - } - }, - { - "name": "created_at", - "ordinal": 13, - "type_info": { - "char_set": 63, - "flags": { - "bits": 1153 - }, - "max_size": 19, - "type": "Timestamp" - } - }, - { - "name": "updated_at", - "ordinal": 14, - "type_info": { - "char_set": 63, - "flags": { - "bits": 9345 - }, - "max_size": 19, - "type": "Timestamp" - } - } - ], - "nullable": [ - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - true, - false, - false - ], - "parameters": { - "Right": 1 - } - }, - "query": "\nSELECT\n jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`,\n\n jobs.status,\n jobs.attempt_count,\n\n jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n jobs.maybe_model_type,\n jobs.maybe_model_token,\n jobs.maybe_raw_inference_text,\n\n jobs.on_success_result_entity_type as maybe_result_entity_type,\n jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n tts_models.title as maybe_tts_model_title,\n voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n jobs.created_at,\n jobs.updated_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\n\nWHERE jobs.token = ?\n " - }, "a993e289a631846abd26185d6a78ba37e10ce98782d00fd4aecb4a4a5502ad38": { "describe": { "columns": [], diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 4a0fcd92d4..6150f3d067 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -12,6 +12,11 @@ pub struct GenericInferenceJobStatus { pub status: String, pub attempt_count: u16, + pub maybe_assigned_worker: Option, + pub maybe_assigned_cluster: Option, + + pub maybe_first_started_at: Option>, + pub request_details: RequestDetails, pub maybe_result_details: Option, @@ -39,6 +44,8 @@ pub struct ResultDetails { /// Whether the location is a full path (for tts) or a hash (for vc) that /// needs to be reconstructed into a path. pub public_bucket_location_is_hash: bool, + + pub maybe_successfully_completed_at: Option>, } /// Look up job status. @@ -72,8 +79,14 @@ SELECT tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path, voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash, + jobs.assigned_worker as maybe_assigned_worker, + jobs.assigned_cluster as maybe_assigned_cluster, + jobs.created_at, - jobs.updated_at + jobs.updated_at, + + jobs.first_started_at as maybe_first_started_at, + jobs.successfully_completed_at as maybe_successfully_completed_at FROM generic_inference_jobs as jobs @@ -126,6 +139,7 @@ WHERE jobs.token = ? entity_token: entity_token.to_string(), public_bucket_location_or_hash: public_bucket_hash.to_string(), public_bucket_location_is_hash: bucket_path_is_hash, + maybe_successfully_completed_at: record.maybe_successfully_completed_at.clone(), } }) }) @@ -135,6 +149,9 @@ WHERE jobs.token = ? job_token: record.job_token, status: record.status, attempt_count: record.attempt_count, + maybe_assigned_worker: record.maybe_assigned_worker, + maybe_assigned_cluster: record.maybe_assigned_cluster, + maybe_first_started_at: record.maybe_first_started_at, request_details: RequestDetails { inference_category: record.inference_category, maybe_model_type: record.maybe_model_type, @@ -168,7 +185,13 @@ struct RawGenericInferenceJobStatus { pub maybe_voice_conversion_public_bucket_hash: Option, // NB: This is the bucket hash. pub maybe_tts_public_bucket_path: Option, // NB: This isn't the bucket path, but the whole hash. + pub maybe_assigned_worker: Option, + pub maybe_assigned_cluster: Option, + pub created_at: DateTime, pub updated_at: DateTime, + + pub maybe_first_started_at: Option>, + pub maybe_successfully_completed_at: Option>, } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 6eb686d8ed..016c417808 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -6,6 +6,7 @@ use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::Path; use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; @@ -27,7 +28,6 @@ use std::borrow::BorrowMut; use std::fmt; use std::ops::Deref; use std::sync::Arc; -use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; use tokens::jobs::inference::InferenceJobToken; /// For the URL PathInfo @@ -75,6 +75,11 @@ pub struct StatusDetailsResponse { /// This can denote inference progress, and the Python code can write to it. pub maybe_extra_status_description: Option, + pub maybe_assigned_worker: Option, + pub maybe_assigned_cluster: Option, + + pub maybe_first_started_at: Option>, + pub attempt_count: u8, } @@ -85,6 +90,8 @@ pub struct ResultDetailsResponse { /// NB: This is only for audio- or video- type results. pub maybe_public_bucket_media_path: Option, + + pub maybe_successfully_completed_at: Option>, } #[derive(Debug)] @@ -176,6 +183,9 @@ pub async fn get_inference_job_status_handler( status: StatusDetailsResponse { status: record.status, maybe_extra_status_description, + maybe_assigned_worker: record.maybe_assigned_worker, + maybe_assigned_cluster: record.maybe_assigned_cluster, + maybe_first_started_at: record.maybe_first_started_at, attempt_count: record.attempt_count as u8, }, maybe_result: record.maybe_result_details.map(|result_details| { @@ -195,6 +205,7 @@ pub async fn get_inference_job_status_handler( entity_type: result_details.entity_type, entity_token: result_details.entity_token, maybe_public_bucket_media_path: Some(public_bucket_media_path), + maybe_successfully_completed_at: result_details.maybe_successfully_completed_at, } }), created_at: record.created_at,