diff --git a/_sql/analytics/generic_inference/generic_inference_jobs.sql b/_sql/analytics/generic_inference/generic_inference_jobs.sql index 1728c876f1..66870a0795 100644 --- a/_sql/analytics/generic_inference/generic_inference_jobs.sql +++ b/_sql/analytics/generic_inference/generic_inference_jobs.sql @@ -135,7 +135,7 @@ UNION WHERE status IN ("started") AND created_at > (CURDATE() - INTERVAL 15 MINUTE) ) as legacy_inner - GROUP BY queue_type + GROUP BY queue_type; -- Generic inference jobs + legacy TTS (this was a production endpoint) -- OLD QUERY: Queues as columns diff --git a/crates/lib/mysql_queries/src/queries/stats/get_unified_queue_stats.rs b/crates/lib/mysql_queries/src/queries/stats/get_unified_queue_stats.rs index 08a15d4139..014f7d6ae4 100644 --- a/crates/lib/mysql_queries/src/queries/stats/get_unified_queue_stats.rs +++ b/crates/lib/mysql_queries/src/queries/stats/get_unified_queue_stats.rs @@ -4,15 +4,15 @@ use errors::AnyhowResult; use sqlx::MySqlPool; #[derive(Serialize, Deserialize, Clone)] -pub struct UnifiedQueueStatsResult { - pub generic_job_count: u64, - pub legacy_tts_job_count: u64, +pub struct QueueStatsRow { + pub queue_type: String, + pub pending_job_count: u64, pub present_time: NaiveDateTime, } -struct QueryResultInternal { - generic_job_count: Option, - legacy_tts_job_count: Option, +struct QueueStatsRowInternal { + queue_type: Option, // FIXME: Shouldn't be nullable + pending_job_count: i64, present_time: NaiveDateTime, } @@ -20,55 +20,63 @@ struct QueryResultInternal { // I'm only doing this because I'm in a hurry. Don't do this. pub async fn get_unified_queue_stats( pool: &MySqlPool, -) -> AnyhowResult> { +) -> AnyhowResult> { // NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale // forever. We currently have no automated means of collecting those jobs. // NB (2): We query the server timestamp so we can cache the results. // The frontend can then monotonically adjust the count based on timestamp by ignoring // past times. - let result : QueryResultInternal = sqlx::query_as!( - QueryResultInternal, + let rows : Vec = sqlx::query_as!( + QueueStatsRowInternal, r#" SELECT - ( + maybe_model_type as queue_type, + count(*) as pending_job_count, + NOW() as present_time + FROM ( SELECT - COUNT(distinct token) as generic_job_count - FROM - ( - SELECT token - FROM generic_inference_jobs - WHERE status = "started" - AND created_at > (CURDATE() - INTERVAL 5 MINUTE) + token, + maybe_model_type + FROM generic_inference_jobs + WHERE status IN ("pending", "attempt_failed") + AND maybe_model_type IS NOT NULL UNION - SELECT token - FROM generic_inference_jobs - WHERE status IN ("pending", "attempt_failed") - ) as g - ) as generic_job_count, - ( SELECT - COUNT(distinct token) as legacy_tts_job_count - FROM - ( - SELECT token - FROM tts_inference_jobs - WHERE status = "started" - AND created_at > (CURDATE() - INTERVAL 5 MINUTE) - UNION - SELECT token - FROM tts_inference_jobs - WHERE status IN ("pending", "attempt_failed") - ) as t - ) as legacy_tts_job_count, - NOW() as present_time; + token, + maybe_model_type + FROM generic_inference_jobs + WHERE status IN ("started") + AND created_at > (CURDATE() - INTERVAL 15 MINUTE) + AND maybe_model_type IS NOT NULL +) as generic_inner +GROUP BY queue_type +UNION +SELECT + "legacy_tts" as queue_type, + count(*) as pending_job_count, + NOW() as present_time +FROM ( + SELECT + token + FROM tts_inference_jobs + WHERE status IN ("pending", "attempt_failed") + UNION + SELECT + token + FROM tts_inference_jobs + WHERE status IN ("started") + AND created_at > (CURDATE() - INTERVAL 15 MINUTE) +) as legacy_inner +GROUP BY queue_type "#, ) - .fetch_one(pool) + .fetch_all(pool) .await?; - Ok(Some(UnifiedQueueStatsResult { - generic_job_count: try_i64_to_u64_or_min(result.generic_job_count.unwrap_or(0)), - legacy_tts_job_count: try_i64_to_u64_or_min(result.legacy_tts_job_count.unwrap_or(0)), - present_time: result.present_time, - })) + Ok(rows.into_iter().map(|row| + QueueStatsRow { + queue_type: row.queue_type.unwrap_or_else(|| "unknown".to_string()), + pending_job_count: try_i64_to_u64_or_min(row.pending_job_count), + present_time: row.present_time, + }).collect::>()) } diff --git a/crates/lib/redis_common/src/redis_cache_keys.rs b/crates/lib/redis_common/src/redis_cache_keys.rs index 0e6d93a2e1..4666a31d31 100644 --- a/crates/lib/redis_common/src/redis_cache_keys.rs +++ b/crates/lib/redis_common/src/redis_cache_keys.rs @@ -10,7 +10,7 @@ impl RedisCacheKeys { } pub fn get_unified_queue_stats_endpoint() -> &'static str { - "q_stats" + "q_stats_2" // NB: Schema changed, hence _2 } pub fn session_record_light(session_token: &str) -> String { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs index d5a66e4998..9884c09b1d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/get_unified_queue_stats.rs @@ -1,12 +1,13 @@ use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; use chrono::NaiveDateTime; use crate::ServerState; +use crate::http_server::endpoints::stats::result_transformer::{CacheableQueueStats, database_result_to_cacheable}; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use errors::AnyhowResult; use hyper::StatusCode; use log::{debug, error, info, warn}; use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::get_pending_inference_job_count; -use mysql_queries::queries::stats::get_unified_queue_stats::{get_unified_queue_stats, UnifiedQueueStatsResult}; +use mysql_queries::queries::stats::get_unified_queue_stats::{get_unified_queue_stats, QueueStatsRow}; use redis_common::redis_cache_keys::RedisCacheKeys; use std::sync::Arc; @@ -106,7 +107,7 @@ pub async fn get_unified_queue_stats_handler( GetUnifiedQueueStatsError::ServerError })?; - let stats_result = match maybe_cached { + let cacheable_stats_result = match maybe_cached { Some(cached) => { debug!("serving from in-memory cache"); cached @@ -120,9 +121,14 @@ pub async fn get_unified_queue_stats_handler( // NB: async closures are not yet stable in Rust, so we include an async block. async move { debug!("querying from database..."); - get_unified_queue_stats( + + let db_result = get_unified_queue_stats( &mysql_pool - ).await + ).await; + + db_result + .map(|db_result| database_result_to_cacheable(db_result)) + .map(|cacheable_result| Some(cacheable_result)) // TODO/FIXME: Make a better redis cache } }; @@ -160,43 +166,35 @@ pub async fn get_unified_queue_stats_handler( } // Happy path... - Ok(Some(stats_result)) => { - server_state.caches.ephemeral.queue_stats.store_copy(&stats_result) + Ok(Some(cacheable_stats)) => { + server_state.caches.ephemeral.queue_stats.store_copy(&cacheable_stats) .map_err(|e| { error!("error storing cache: {:?}", e); GetUnifiedQueueStatsError::ServerError })?; - - stats_result + cacheable_stats } - Ok(None) => { - // NB: This should be impossible since the DB always returns "Some" - // (We artificially wrapped the result in Option.) - UnifiedQueueStatsResult { - generic_job_count: 10_000, - legacy_tts_job_count: 10_000, - present_time: NaiveDateTime::from_timestamp(0, 0), - } - } + // TODO/FIXME(bt,2023-07-30): This match arm should never happen. + Ok(None) => CacheableQueueStats::default() } }, }; render_response_ok(GetUnifiedQueueStatsSuccessResponse { success: true, - cache_time: stats_result.present_time, + cache_time: cacheable_stats_result.cache_time, refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis, inference: ModernInferenceQueueStats { - total_pending_job_count: stats_result.generic_job_count, - pending_job_count: stats_result.generic_job_count, + total_pending_job_count: cacheable_stats_result.queues.total_generic, + pending_job_count: cacheable_stats_result.queues.total_generic, by_queue: ByQueueStats { - pending_svc_jobs: 0, - pending_rvc_jobs: 0, + pending_svc_jobs: cacheable_stats_result.queues.so_vits_svc, + pending_rvc_jobs: cacheable_stats_result.queues.rvc_v2, } }, legacy_tts: LegacyQueueDetails { - pending_job_count: stats_result.legacy_tts_job_count + pending_job_count: cacheable_stats_result.queues.legacy_tts, }, }) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/mod.rs index 3e0199e980..9f0eb8062b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/mod.rs @@ -1 +1,2 @@ -pub mod get_unified_queue_stats; \ No newline at end of file +pub mod get_unified_queue_stats; +pub mod result_transformer; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs new file mode 100644 index 0000000000..d19effa76f --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/stats/result_transformer.rs @@ -0,0 +1,58 @@ +use chrono::NaiveDateTime; +use mysql_queries::queries::stats::get_unified_queue_stats::QueueStatsRow; +use std::collections::HashMap; + +#[derive(Serialize, Deserialize, Clone)] +pub struct CacheableQueueStats { + pub cache_time: NaiveDateTime, + pub queues: Queues, +} + +impl Default for CacheableQueueStats { + fn default() -> Self { + Self { + cache_time: NaiveDateTime::from_timestamp(0, 0), + queues: Default::default(), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct Queues { + pub legacy_tts: u64, // Tacotron2 + pub total_generic: u64, // Everything except Tacotron2 summed together + pub rvc_v2: u64, + pub so_vits_svc: u64, +} + +pub fn database_result_to_cacheable(database_records: Vec) -> CacheableQueueStats { + let database_time = database_records.first() + .map(|record| record.present_time) + .unwrap_or(NaiveDateTime::from_timestamp(0, 0)); + + // Queue lengths + let mut queue_lengths : HashMap = HashMap::new(); + + for record in database_records.into_iter() { + queue_lengths.insert(record.queue_type, record.pending_job_count); + } + + let rvc_v2 = unwrap(queue_lengths.get("rvc_v2")); + let so_vits_svc = unwrap(queue_lengths.get("so_vits_svc")); + + let total_generic = rvc_v2 + so_vits_svc; + + CacheableQueueStats { + cache_time: database_time, + queues: Queues { + legacy_tts: unwrap(queue_lengths.get("legacy_tts")), + total_generic, + rvc_v2, + so_vits_svc, + }, + } +} + +fn unwrap(inner: Option<&u64>) -> u64 { + inner.map(|i|*i).unwrap_or(0) +} diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index 9ccb1edd5d..9b146490ce 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -6,9 +6,11 @@ use crate::StaticApiTokenSet; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken; use crate::http_server::endpoints::categories::tts::list_tts_categories::DisplayCategory; use crate::http_server::endpoints::leaderboard::get_leaderboard::LeaderboardInfo; +use crate::http_server::endpoints::stats::result_transformer::CacheableQueueStats; use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse; use crate::http_server::endpoints::voice_conversion::models::list_voice_conversion_models::VoiceConversionModel; use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; +use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; use crate::util::encrypted_sort_id::SortKeyCrypto; use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; @@ -17,7 +19,6 @@ use mysql_queries::mediators::badge_granter::BadgeGranter; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult; use mysql_queries::queries::model_categories::list_categories_query_builder::CategoryList; -use mysql_queries::queries::stats::get_unified_queue_stats::UnifiedQueueStatsResult; use mysql_queries::queries::tts::tts_inference_jobs::get_pending_tts_inference_job_count::TtsQueueLengthResult; use mysql_queries::queries::w2l::w2l_templates::list_w2l_templates::W2lTemplateRecordForList; use r2d2_redis::{r2d2, RedisConnectionManager}; @@ -27,7 +28,6 @@ use sqlx::MySqlPool; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use users_component::utils::session_checker::SessionChecker; use users_component::utils::session_cookie_manager::SessionCookieManager; -use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache; /// State that is injected into every endpoint. pub struct ServerState { @@ -173,7 +173,7 @@ pub struct EphemeralInMemoryCaches { /// Stats on generic inference queue and legacy TTS queue (combined). /// The frontend will consult a distributed cache and use the monotonic DB time as a /// vector clock. - pub queue_stats: SingleItemTtlCache, + pub queue_stats: SingleItemTtlCache, /// Generic inference queue length /// The frontend will consult a distributed cache and use the monotonic DB time as a