Enrich stats query with per-queue info

This commit is contained in:
Brandon Thomas committed 2023-07-30 18:12:45 -04:00
1 parent e8be6f9775
commit ffb081777d
7 files changed
+137 -72

No files matched your search

@@ -135,7 +135,7 @@ UNION
WHERE status IN ("started")
AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
) as legacy_inner
GROUP BY queue_type
GROUP BY queue_type;
-- Generic inference jobs + legacy TTS (this was a production endpoint)
-- OLD QUERY: Queues as columns
@@ -4,15 +4,15 @@ use errors::AnyhowResult;
use sqlx::MySqlPool;
#[derive(Serialize, Deserialize, Clone)]
pub struct UnifiedQueueStatsResult {
pub generic_job_count: u64,
pub legacy_tts_job_count: u64,
pub struct QueueStatsRow {
pub queue_type: String,
pub pending_job_count: u64,
pub present_time: NaiveDateTime,
}
struct QueryResultInternal {
generic_job_count: Option<i64>,
legacy_tts_job_count: Option<i64>,
struct QueueStatsRowInternal {
queue_type: Option<String>, // FIXME: Shouldn't be nullable
pending_job_count: i64,
present_time: NaiveDateTime,
}
@@ -20,55 +20,63 @@ struct QueryResultInternal {
// I'm only doing this because I'm in a hurry. Don't do this.
pub async fn get_unified_queue_stats(
pool: &MySqlPool,
) -> AnyhowResult<Option<UnifiedQueueStatsResult>> {
) -> AnyhowResult<Vec<QueueStatsRow>> {
// NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale
// forever. We currently have no automated means of collecting those jobs.
// NB (2): We query the server timestamp so we can cache the results.
// The frontend can then monotonically adjust the count based on timestamp by ignoring
// past times.
let result : QueryResultInternal = sqlx::query_as!(
QueryResultInternal,
let rows : Vec<QueueStatsRowInternal> = sqlx::query_as!(
QueueStatsRowInternal,
r#"
SELECT
(
maybe_model_type as queue_type,
count(*) as pending_job_count,
NOW() as present_time
FROM (
SELECT
COUNT(distinct token) as generic_job_count
FROM
(
SELECT token
FROM generic_inference_jobs
WHERE status = "started"
AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
token,
maybe_model_type
FROM generic_inference_jobs
WHERE status IN ("pending", "attempt_failed")
AND maybe_model_type IS NOT NULL
UNION
SELECT token
FROM generic_inference_jobs
WHERE status IN ("pending", "attempt_failed")
) as g
) as generic_job_count,
(
SELECT
COUNT(distinct token) as legacy_tts_job_count
FROM
(
SELECT token
FROM tts_inference_jobs
WHERE status = "started"
AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
UNION
SELECT token
FROM tts_inference_jobs
WHERE status IN ("pending", "attempt_failed")
) as t
) as legacy_tts_job_count,
NOW() as present_time;
token,
maybe_model_type
FROM generic_inference_jobs
WHERE status IN ("started")
AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
AND maybe_model_type IS NOT NULL
) as generic_inner
GROUP BY queue_type
UNION
SELECT
"legacy_tts" as queue_type,
count(*) as pending_job_count,
NOW() as present_time
FROM (
SELECT
token
FROM tts_inference_jobs
WHERE status IN ("pending", "attempt_failed")
UNION
SELECT
token
FROM tts_inference_jobs
WHERE status IN ("started")
AND created_at > (CURDATE() - INTERVAL 15 MINUTE)
) as legacy_inner
GROUP BY queue_type
"#,
)
.fetch_one(pool)
.fetch_all(pool)
.await?;
Ok(Some(UnifiedQueueStatsResult {
generic_job_count: try_i64_to_u64_or_min(result.generic_job_count.unwrap_or(0)),
legacy_tts_job_count: try_i64_to_u64_or_min(result.legacy_tts_job_count.unwrap_or(0)),
present_time: result.present_time,
}))
Ok(rows.into_iter().map(|row|
QueueStatsRow {
queue_type: row.queue_type.unwrap_or_else(|| "unknown".to_string()),
pending_job_count: try_i64_to_u64_or_min(row.pending_job_count),
present_time: row.present_time,
}).collect::<Vec<_>>())
}
@@ -10,7 +10,7 @@ impl RedisCacheKeys {
}
pub fn get_unified_queue_stats_endpoint() -> &'static str {
"q_stats"
"q_stats_2" // NB: Schema changed, hence _2
}
pub fn session_record_light(session_token: &str) -> String {
@@ -1,12 +1,13 @@
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
use chrono::NaiveDateTime;
use crate::ServerState;
use crate::http_server::endpoints::stats::result_transformer::{CacheableQueueStats, database_result_to_cacheable};
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use errors::AnyhowResult;
use hyper::StatusCode;
use log::{debug, error, info, warn};
use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::get_pending_inference_job_count;
use mysql_queries::queries::stats::get_unified_queue_stats::{get_unified_queue_stats, UnifiedQueueStatsResult};
use mysql_queries::queries::stats::get_unified_queue_stats::{get_unified_queue_stats, QueueStatsRow};
use redis_common::redis_cache_keys::RedisCacheKeys;
use std::sync::Arc;
@@ -106,7 +107,7 @@ pub async fn get_unified_queue_stats_handler(
GetUnifiedQueueStatsError::ServerError
})?;
let stats_result = match maybe_cached {
let cacheable_stats_result = match maybe_cached {
Some(cached) => {
debug!("serving from in-memory cache");
cached
@@ -120,9 +121,14 @@ pub async fn get_unified_queue_stats_handler(
// NB: async closures are not yet stable in Rust, so we include an async block.
async move {
debug!("querying from database...");
get_unified_queue_stats(
let db_result = get_unified_queue_stats(
&mysql_pool
).await
).await;
db_result
.map(|db_result| database_result_to_cacheable(db_result))
.map(|cacheable_result| Some(cacheable_result)) // TODO/FIXME: Make a better redis cache
}
};
@@ -160,43 +166,35 @@ pub async fn get_unified_queue_stats_handler(
}
// Happy path...
Ok(Some(stats_result)) => {
server_state.caches.ephemeral.queue_stats.store_copy(&stats_result)
Ok(Some(cacheable_stats)) => {
server_state.caches.ephemeral.queue_stats.store_copy(&cacheable_stats)
.map_err(|e| {
error!("error storing cache: {:?}", e);
GetUnifiedQueueStatsError::ServerError
})?;
stats_result
cacheable_stats
}
Ok(None) => {
// NB: This should be impossible since the DB always returns "Some"
// (We artificially wrapped the result in Option<T>.)
UnifiedQueueStatsResult {
generic_job_count: 10_000,
legacy_tts_job_count: 10_000,
present_time: NaiveDateTime::from_timestamp(0, 0),
}
}
// TODO/FIXME(bt,2023-07-30): This match arm should never happen.
Ok(None) => CacheableQueueStats::default()
}
},
};
render_response_ok(GetUnifiedQueueStatsSuccessResponse {
success: true,
cache_time: stats_result.present_time,
cache_time: cacheable_stats_result.cache_time,
refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis,
inference: ModernInferenceQueueStats {
total_pending_job_count: stats_result.generic_job_count,
pending_job_count: stats_result.generic_job_count,
total_pending_job_count: cacheable_stats_result.queues.total_generic,
pending_job_count: cacheable_stats_result.queues.total_generic,
by_queue: ByQueueStats {
pending_svc_jobs: 0,
pending_rvc_jobs: 0,
pending_svc_jobs: cacheable_stats_result.queues.so_vits_svc,
pending_rvc_jobs: cacheable_stats_result.queues.rvc_v2,
}
},
legacy_tts: LegacyQueueDetails {
pending_job_count: stats_result.legacy_tts_job_count
pending_job_count: cacheable_stats_result.queues.legacy_tts,
},
})
}
@@ -1 +1,2 @@
pub mod get_unified_queue_stats;
pub mod get_unified_queue_stats;
pub mod result_transformer;
@@ -0,0 +1,58 @@
use chrono::NaiveDateTime;
use mysql_queries::queries::stats::get_unified_queue_stats::QueueStatsRow;
use std::collections::HashMap;
#[derive(Serialize, Deserialize, Clone)]
pub struct CacheableQueueStats {
pub cache_time: NaiveDateTime,
pub queues: Queues,
}
impl Default for CacheableQueueStats {
fn default() -> Self {
Self {
cache_time: NaiveDateTime::from_timestamp(0, 0),
queues: Default::default(),
}
}
}
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct Queues {
pub legacy_tts: u64, // Tacotron2
pub total_generic: u64, // Everything except Tacotron2 summed together
pub rvc_v2: u64,
pub so_vits_svc: u64,
}
pub fn database_result_to_cacheable(database_records: Vec<QueueStatsRow>) -> CacheableQueueStats {
let database_time = database_records.first()
.map(|record| record.present_time)
.unwrap_or(NaiveDateTime::from_timestamp(0, 0));
// Queue lengths
let mut queue_lengths : HashMap<String, u64> = HashMap::new();
for record in database_records.into_iter() {
queue_lengths.insert(record.queue_type, record.pending_job_count);
}
let rvc_v2 = unwrap(queue_lengths.get("rvc_v2"));
let so_vits_svc = unwrap(queue_lengths.get("so_vits_svc"));
let total_generic = rvc_v2 + so_vits_svc;
CacheableQueueStats {
cache_time: database_time,
queues: Queues {
legacy_tts: unwrap(queue_lengths.get("legacy_tts")),
total_generic,
rvc_v2,
so_vits_svc,
},
}
}
fn unwrap(inner: Option<&u64>) -> u64 {
inner.map(|i|*i).unwrap_or(0)
}
@@ -6,9 +6,11 @@ use crate::StaticApiTokenSet;
use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken;
use crate::http_server::endpoints::categories::tts::list_tts_categories::DisplayCategory;
use crate::http_server::endpoints::leaderboard::get_leaderboard::LeaderboardInfo;
use crate::http_server::endpoints::stats::result_transformer::CacheableQueueStats;
use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForResponse;
use crate::http_server::endpoints::voice_conversion::models::list_voice_conversion_models::VoiceConversionModel;
use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter;
use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache;
use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus;
use crate::util::encrypted_sort_id::SortKeyCrypto;
use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList;
@@ -17,7 +19,6 @@ use mysql_queries::mediators::badge_granter::BadgeGranter;
use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult;
use mysql_queries::queries::model_categories::list_categories_query_builder::CategoryList;
use mysql_queries::queries::stats::get_unified_queue_stats::UnifiedQueueStatsResult;
use mysql_queries::queries::tts::tts_inference_jobs::get_pending_tts_inference_job_count::TtsQueueLengthResult;
use mysql_queries::queries::w2l::w2l_templates::list_w2l_templates::W2lTemplateRecordForList;
use r2d2_redis::{r2d2, RedisConnectionManager};
@@ -27,7 +28,6 @@ use sqlx::MySqlPool;
use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
use users_component::utils::session_checker::SessionChecker;
use users_component::utils::session_cookie_manager::SessionCookieManager;
use crate::memory_cache::model_token_to_info_cache::ModelTokenToInfoCache;
/// State that is injected into every endpoint.
pub struct ServerState {
@@ -173,7 +173,7 @@ pub struct EphemeralInMemoryCaches {
/// Stats on generic inference queue and legacy TTS queue (combined).
/// The frontend will consult a distributed cache and use the monotonic DB time as a
/// vector clock.
pub queue_stats: SingleItemTtlCache<UnifiedQueueStatsResult>,
pub queue_stats: SingleItemTtlCache<CacheableQueueStats>,
/// Generic inference queue length
/// The frontend will consult a distributed cache and use the monotonic DB time as a