diff --git a/_sql/migrations_squashed/generic_inference_jobs.sql b/_sql/migrations_squashed/generic_inference_jobs.sql index c998a2c98e..57bd9a0625 100644 --- a/_sql/migrations_squashed/generic_inference_jobs.sql +++ b/_sql/migrations_squashed/generic_inference_jobs.sql @@ -17,12 +17,16 @@ CREATE TABLE generic_inference_jobs ( -- ========== INFERENCE DETAILS ========== + -- TODO: Rename this inference_category and add `maybe_model_type` -- Type of inference -- Examples (may not be up to date): -- * text_to_speech -- * voice_conversion inference_type VARCHAR(32) NOT NULL, + -- TODO: Add `maybe_model_type` here + -- TODO: Move maybe_model_token here + -- Polymorphic arguments payload that depends on the type of inference job. -- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters. maybe_inference_args TEXT DEFAULT NULL, @@ -87,6 +91,7 @@ CREATE TABLE generic_inference_jobs ( -- - (There will be future levels for paid Twitch and social FakeYou rewards.) priority_level SMALLINT UNSIGNED NOT NULL DEFAULT 0, + -- TODO: add is_for_storyteller_product is_from_premium_user BOOLEAN NOT NULL DEFAULT FALSE, is_from_api_user BOOLEAN NOT NULL DEFAULT FALSE, is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE, diff --git a/crates/lib/actix_helpers/src/route_builder.rs b/crates/lib/actix_helpers/src/route_builder.rs index 4c81e76c25..b4752aad03 100644 --- a/crates/lib/actix_helpers/src/route_builder.rs +++ b/crates/lib/actix_helpers/src/route_builder.rs @@ -47,7 +47,7 @@ impl RouteBuilder self.app } - /// Add an HTTP GET route. + /// Add an HTTP GET route. This also adds the HEAD request for CORS. pub fn add_get(mut self, path: &str, handler: F) -> Self where F: Handler, @@ -63,7 +63,7 @@ impl RouteBuilder self } - /// Add an HTTP POST route. + /// Add an HTTP POST route. This also adds the HEAD request for CORS. pub fn add_post(mut self, path: &str, handler: F) -> Self where F: Handler, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_generic_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_generic_inference_job_status.rs deleted file mode 100644 index 536ec7f62d..0000000000 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_generic_inference_job_status.rs +++ /dev/null @@ -1,59 +0,0 @@ -use anyhow::anyhow; -use chrono::{DateTime, Utc}; -use errors::AnyhowResult; -use log::warn; -use sqlx::MySqlPool; -use tokens::jobs::inference::InferenceJobToken; - -pub struct GenericInferenceJobStatus { - pub job_token: InferenceJobToken, - - pub status: String, - pub attempt_count: u16, - - pub maybe_result_entity_type: Option, - pub maybe_result_entity_token: Option, - - pub created_at: DateTime, - pub updated_at: DateTime, -} - -/// Look up job status. -/// Returns Ok(None) when the record cannot be found. -pub async fn get_generic_inference_job_status(job_token: &InferenceJobToken, mysql_pool: &MySqlPool) - -> AnyhowResult> -{ - let maybe_status = sqlx::query_as!( - GenericInferenceJobStatus, - r#" -SELECT - jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`, - - jobs.status, - jobs.attempt_count, - jobs.on_success_result_entity_type as maybe_result_entity_type, - jobs.on_success_result_entity_token as maybe_result_entity_token, - - jobs.created_at, - jobs.updated_at - -FROM generic_inference_jobs as jobs - -WHERE jobs.token = ? - "#, - job_token - ) - .fetch_one(mysql_pool) - .await; - - match maybe_status { - Ok(record) => Ok(Some(record)), - Err(err) => match err { - sqlx::Error::RowNotFound => Ok(None), - _ => { - warn!("error querying job record: {:?}", err); - Err(anyhow!("error querying job record: {:?}", err)) - } - } - } -} diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs new file mode 100644 index 0000000000..ef28338610 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -0,0 +1,124 @@ +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use errors::AnyhowResult; +use log::warn; +use sqlx::MySqlPool; +use tokens::jobs::inference::InferenceJobToken; + +pub struct GenericInferenceJobStatus { + pub job_token: InferenceJobToken, + + pub status: String, + pub attempt_count: u16, + + pub request_details: RequestDetails, + pub maybe_result_details: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub struct RequestDetails { + // TODO: Rename this to inference category! + pub inference_type: String, + pub maybe_model_token: Option, + + /// TTS input. In the future, perhaps voice conversion SST + pub maybe_raw_inference_text: Option, +} + +pub struct ResultDetails { + pub entity_type: String, + pub entity_token: String, +} + +/// Look up job status. +/// Returns Ok(None) when the record cannot be found. +pub async fn get_inference_job_status(job_token: &InferenceJobToken, mysql_pool: &MySqlPool) + -> AnyhowResult> +{ + let maybe_status = sqlx::query_as!( + RawGenericInferenceJobStatus, + r#" +SELECT + jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`, + + jobs.status, + jobs.attempt_count, + + jobs.inference_type, + jobs.maybe_model_token, + jobs.maybe_raw_inference_text, + + jobs.on_success_result_entity_type as maybe_result_entity_type, + jobs.on_success_result_entity_token as maybe_result_entity_token, + + jobs.created_at, + jobs.updated_at + +FROM generic_inference_jobs as jobs + +WHERE jobs.token = ? + "#, + job_token + ) + .fetch_one(mysql_pool) + .await; + + let record = match maybe_status { + Ok(record) => record, + Err(err) => match err { + sqlx::Error::RowNotFound => return Ok(None), + _ => { + warn!("error querying job record: {:?}", err); + return Err(anyhow!("error querying job record: {:?}", err)); + } + } + }; + + let maybe_result_details = record + .maybe_result_entity_type + .as_deref() + .and_then(|entity_type| { + record.maybe_result_entity_token + .as_deref() + .map(|entity_token| { + ResultDetails { + entity_type: entity_type.to_string(), + entity_token: entity_token.to_string(), + } + }) + }); + + Ok(Some(GenericInferenceJobStatus { + job_token: record.job_token, + status: record.status, + attempt_count: record.attempt_count, + request_details: RequestDetails { + inference_type: record.inference_type, + maybe_model_token: record.maybe_model_token, + maybe_raw_inference_text: record.maybe_raw_inference_text, + }, + maybe_result_details, + created_at: record.created_at, + updated_at: record.updated_at, + })) +} + +struct RawGenericInferenceJobStatus { + pub job_token: InferenceJobToken, + + pub status: String, + pub attempt_count: u16, + + pub inference_type: String, + pub maybe_model_token: Option, + pub maybe_raw_inference_text: Option, + + pub maybe_result_entity_type: Option, + pub maybe_result_entity_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_pending_inference_job_count.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_pending_inference_job_count.rs new file mode 100644 index 0000000000..6da30b487c --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_pending_inference_job_count.rs @@ -0,0 +1,51 @@ +use chrono::NaiveDateTime; +use crate::helpers::numeric_converters::try_i64_to_u64_or_min; +use errors::AnyhowResult; +use sqlx::MySqlPool; + +#[derive(Clone)] +pub struct InferenceQueueLengthResult { + pub record_count: u64, + pub present_time: NaiveDateTime, +} + +struct QueryResultInternal { + record_count: i64, + present_time: NaiveDateTime, +} + +pub async fn get_pending_inference_job_count( + pool: &MySqlPool, +) -> AnyhowResult { + // NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale + // forever. We currently have no automated means of collecting those jobs. + // NB (2): We query the server timestamp so we can cache the results. + // The frontend can then monotonically adjust the count based on timestamp by ignoring + // past times. + let result : QueryResultInternal = sqlx::query_as!( + QueryResultInternal, + r#" +SELECT + COUNT(distinct token) as record_count, + NOW() as present_time +FROM +( + SELECT token + FROM generic_inference_jobs + WHERE status = "started" + AND created_at > (CURDATE() - INTERVAL 5 MINUTE) +UNION + SELECT token + FROM generic_inference_jobs + WHERE status IN ("pending", "attempt_failed") +) as t + "#, + ) + .fetch_one(pool) + .await?; + + Ok(InferenceQueueLengthResult { + record_count: try_i64_to_u64_or_min(result.record_count), + present_time: result.present_time, + }) +} diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs index 3e4d73ced2..71a2786c0b 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs @@ -1,2 +1,3 @@ -pub mod get_generic_inference_job_status; +pub mod get_inference_job_status; +pub mod get_pending_inference_job_count; pub mod insert_generic_inference_job; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs new file mode 100644 index 0000000000..96a8cf2134 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -0,0 +1,204 @@ +use actix_http::Error; +use actix_http::http::header; +use actix_web::HttpResponseBuilder; +use actix_web::cookie::Cookie; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; +use chrono::{DateTime, Utc}; +use crate::AnyhowResult; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; +use log::{info, warn, log, error}; +use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status; +use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::{Pool, PooledConnection}; +use r2d2_redis::redis::{Commands, RedisError, RedisResult}; +use redis_common::redis_keys::RedisKeys; +use regex::Regex; +use sqlx::MySqlPool; +use sqlx::error::DatabaseError; +use sqlx::error::Error::Database; +use sqlx::mysql::MySqlDatabaseError; +use std::borrow::BorrowMut; +use std::fmt; +use std::ops::Deref; +use std::sync::Arc; +use tokens::jobs::inference::InferenceJobToken; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetInferenceJobStatusPathInfo { + token: InferenceJobToken, +} + +#[derive(Serialize)] +pub struct InferenceJobStatusResponsePayload { + pub job_token: InferenceJobToken, + + pub request: RequestDetailsResponse, + pub status: StatusDetailsResponse, + pub maybe_result: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Serialize)] +pub struct RequestDetailsResponse { + // TODO: Rename this inference_category (!!!) and add model_type. + pub inference_category: String, + pub maybe_model_type: Option, + pub maybe_model_token: Option, + + /// Title of the model + pub title: String, + + /// If the result was TTS, this is the raw inference text. + pub maybe_raw_inference_text: Option, +} + +#[derive(Serialize)] +pub struct StatusDetailsResponse { + /// Primary status from the database (a state machine). + pub status: String, + + /// Extra, temporary status from Redis. + /// This can denote inference progress, and the Python code can write to it. + pub maybe_extra_status_description: Option, + + pub attempt_count: u8, +} + +#[derive(Serialize)] +pub struct ResultDetailsResponse { + pub entity_type: String, + pub entity_token: String, + + /// NB: This is only for audio- or video- type results. + pub maybe_public_bucket_wav_media_path: Option, +} + +#[derive(Serialize)] +pub struct GetInferenceJobStatusSuccessResponse { + pub success: bool, + pub state: InferenceJobStatusResponsePayload, +} + +#[derive(Debug)] +pub enum GetInferenceJobStatusError { + ServerError, + NotFound, +} + +impl ResponseError for GetInferenceJobStatusError { + fn status_code(&self) -> StatusCode { + match *self { + GetInferenceJobStatusError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + GetInferenceJobStatusError::NotFound => StatusCode::NOT_FOUND, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + Self::ServerError => "server error".to_string(), + Self::NotFound => "not found".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetInferenceJobStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + + +pub async fn get_inference_job_status_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + if path.token.as_str().trim() == "None" { + // NB: A bunch of Python clients use our API and can fail in this manner. + // This was a large traffic driver during the 2023-03-08 outage. + return Err(GetInferenceJobStatusError::NotFound); + } + + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_status = get_inference_job_status(&path.token, &server_state.mysql_pool).await; + + let record = match maybe_status { + Ok(Some(record)) => record, + Ok(None) => return Err(GetInferenceJobStatusError::NotFound), + Err(err) => { + error!("tts job query error: {:?}", err); + return Err(GetInferenceJobStatusError::ServerError); + } + }; + + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + error!("redis error: {:?}", e); + GetInferenceJobStatusError::ServerError + })?; + + let extra_status_key = RedisKeys::generic_inference_extra_status_info(path.token.as_str()); + let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { + Ok(Some(status)) => { + Some(status) + }, + Ok(None) => None, + Err(e) => { + error!("redis error: {:?}", e); + None // Fail open + }, + }; + + let record_for_response = InferenceJobStatusResponsePayload { + job_token: record.job_token, + request: RequestDetailsResponse { + inference_category: record.request_details.inference_type, + maybe_model_type: None, // TODO: This will require a new field + maybe_model_token: record.request_details.maybe_model_token, + title: "AI Model".to_string(), // TODO: This will require polymorphic joins + maybe_raw_inference_text: record.request_details.maybe_raw_inference_text, + }, + status: StatusDetailsResponse { + status: record.status, + maybe_extra_status_description, + attempt_count: record.attempt_count as u8, + }, + maybe_result: record.maybe_result_details.map(|result_details| { + ResultDetailsResponse { + entity_type: result_details.entity_type, + entity_token: result_details.entity_token, + maybe_public_bucket_wav_media_path: None, // TODO: This will require a new field + } + }), + created_at: record.created_at, + updated_at: record.updated_at, + }; + + let response = GetInferenceJobStatusSuccessResponse { + success: true, + state: record_for_response, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| { + error!("error returning response: {:?}", e); + GetInferenceJobStatusError::ServerError + })?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_pending_inference_job_count.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_pending_inference_job_count.rs new file mode 100644 index 0000000000..22042ec2b8 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_pending_inference_job_count.rs @@ -0,0 +1,146 @@ +use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; +use chrono::NaiveDateTime; +use crate::ServerState; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use errors::AnyhowResult; +use hyper::StatusCode; +use log::{debug, error, info, warn}; +use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::get_pending_inference_job_count; +use std::fmt; +use std::sync::Arc; + +#[derive(Serialize)] +pub struct GetPendingInferenceJobSuccessResponse { + pub success: bool, + pub pending_job_count: u64, + pub cache_time: NaiveDateTime, + + /// Tell the frontend client how fast to refresh their view of this list. + /// During an attack, we may want this to go extremely slow. + pub refresh_interval_millis: u64, +} + +#[derive(Debug)] +pub enum GetPendingInferenceJobCountError { + ServerError, +} + +impl ResponseError for GetPendingInferenceJobCountError { + fn status_code(&self) -> StatusCode { + match *self { + GetPendingInferenceJobCountError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetPendingInferenceJobCountError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetPendingInferenceJobCountError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn get_pending_inference_job_count_handler( + http_request: HttpRequest, + server_state: web::Data> +) -> Result { + + if server_state.flags.disable_inference_queue_length_endpoint { + // NB: Despite the cache being a powerful protector of the database (this is an expensive query), + // if the cache goes stale during an outage, there is no protection. This feature flag lets us + // shut off all traffic to the endpoint. + return render_response_busy(GetPendingInferenceJobSuccessResponse { + success: true, + pending_job_count: 10_000, + cache_time: NaiveDateTime::from_timestamp(0, 0), + refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis, + }); + } + + let maybe_cached = server_state.caches.inference_queue_length.grab_copy_without_bump_if_unexpired() + .map_err(|e| { + error!("error consulting cache: {:?}", e); + GetPendingInferenceJobCountError::ServerError + })?; + + let count_result = match maybe_cached { + Some(cached) => { + cached + }, + None => { + debug!("populating generic inference queue length from database"); + + let count_query_result = get_pending_inference_job_count( + &server_state.mysql_pool + ).await; + + match count_query_result { + // If the database misbehaves (eg. DDoS), let's stop spamming it. + // We'll attempt to read the old value from the cache and keep going. + Err(err) => { + warn!("error querying database / inserting into cache: {:?}", err); + + let maybe_cached = server_state.caches.inference_queue_length.grab_even_expired_and_bump() + .map_err(|err| { + error!("error consulting cache (even expired): {:?}", err); + GetPendingInferenceJobCountError::ServerError + })?; + + maybe_cached.ok_or_else(|| { + error!("error querying database and subsequently reading cache: {:?}", err); + GetPendingInferenceJobCountError::ServerError + })? + } + + // Happy path... + Ok(count_result) => { + server_state.caches.inference_queue_length.store_copy(&count_result) + .map_err(|e| { + error!("error storing cache: {:?}", e); + GetPendingInferenceJobCountError::ServerError + })?; + + count_result + } + } + }, + }; + + render_response_ok(GetPendingInferenceJobSuccessResponse { + success: true, + pending_job_count: count_result.record_count, + cache_time: count_result.present_time, + refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis, + }) +} + +pub fn render_response_busy(response: GetPendingInferenceJobSuccessResponse) -> Result { + let body = render_response_payload(response)?; + Ok(HttpResponse::TooManyRequests() + .content_type("application/json") + .body(body)) +} + +pub fn render_response_ok(response: GetPendingInferenceJobSuccessResponse) -> Result { + let body = render_response_payload(response)?; + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +pub fn render_response_payload(response: GetPendingInferenceJobSuccessResponse) -> Result { + let body = serde_json::to_string(&response) + .map_err(|e| { + error!("error returning response: {:?}", e); + GetPendingInferenceJobCountError::ServerError + })?; + Ok(body) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs new file mode 100644 index 0000000000..33c74fd282 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/mod.rs @@ -0,0 +1,2 @@ +pub mod get_inference_job_status; +pub mod get_pending_inference_job_count; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 8b4cee3e77..d27ddc1b3a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -4,6 +4,7 @@ pub mod comments; pub mod download_job; pub mod events; pub mod flags; +pub mod inference_job; pub mod investor_demo; pub mod leaderboard; pub mod media_uploads; diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 8f7bca45df..85ef72abef 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -296,6 +296,10 @@ async fn main() -> AnyhowResult<()> { easyenv::get_env_duration_seconds_or_default("LEADERBOARD_CACHE_TTL_SECONDS", Duration::from_secs(60)) ); + let inference_queue_length_cache = SingleItemTtlCache::create_with_duration( + easyenv::get_env_duration_seconds_or_default("INFERENCE_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30)) + ); + // NB: This secret really isn't too important. // We can even rotate it without too much impact to users. let sort_key_crypto_secret = @@ -370,8 +374,10 @@ async fn main() -> AnyhowResult<()> { let service_feature_flags = StaticFeatureFlags { // Permanent (control plane / safety) flags global_429_pushback_filter_enabled: easyenv::get_env_bool_or_default("FF_GLOBAL_429_PUSHBACK_FILTER_ENABLED", false), + disable_inference_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_INFERENCE_QUEUE_LENGTH_ENDPOINT", false), disable_tts_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_QUEUE_LENGTH_ENDPOINT", false), disable_tts_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_MODEL_LIST_ENDPOINT", false), + frontend_pending_inference_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_INFERENCE_REFRESH_INTERVAL_MILLIS", 15_000)?, frontend_pending_tts_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_TTS_REFRESH_INTERVAL_MILLIS", 15_000)?, troll_ban_user_percent: easyenv::get_env_num("FF_TROLL_BANNED_USER_PERCENT", 0)?, @@ -438,6 +444,7 @@ async fn main() -> AnyhowResult<()> { tts_queue_length: tts_queue_length_cache, tts_model_category_assignments: tts_model_category_assignments_cache, leaderboard: leaderboard_cache, + inference_queue_length: inference_queue_length_cache, }, twitch_oauth: TwitchOauth { secrets: TwitchOauthSecrets { diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index fcb442fe68..6479b1a93a 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -23,6 +23,7 @@ use crate::http_server::endpoints::download_job::get_generic_upload_job_status:: use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler; use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler; +use crate::http_server::endpoints::inference_job::get_pending_inference_job_count::get_pending_inference_job_count_handler; use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler; use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; @@ -110,6 +111,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use users_component::default_routes::add_suggested_api_v1_account_creation_and_session_routes; use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; +use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler; pub fn add_routes (app: App) -> App where @@ -149,6 +151,13 @@ pub fn add_routes (app: App) -> App .add_post("/v1/comments/delete/{comment_token}", delete_comment_handler) .into_app(); + // ==================== "Generic" Inference ==================== + + let mut app = RouteBuilder::from_app(app) + .add_get("/v1/model_inference/job_status/{token}", get_inference_job_status_handler) + .add_get("/v1/model_inference/queue_length", get_pending_inference_job_count_handler) + .into_app(); + // ==================== COMPONENTS ==================== app = add_suggested_api_v1_account_creation_and_session_routes(app); // /create_account, /session, /login, /logout diff --git a/crates/service/web/storyteller_web/src/server_state.rs b/crates/service/web/storyteller_web/src/server_state.rs index cbc92b7a06..3348f76951 100644 --- a/crates/service/web/storyteller_web/src/server_state.rs +++ b/crates/service/web/storyteller_web/src/server_state.rs @@ -9,12 +9,14 @@ use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForRespon use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; use crate::util::encrypted_sort_id::SortKeyCrypto; +use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use mysql_queries::mediators::badge_granter::BadgeGranter; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; +use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult; use mysql_queries::queries::model_categories::list_categories_query_builder::CategoryList; use mysql_queries::queries::tts::tts_inference_jobs::get_pending_tts_inference_job_count::TtsQueueLengthResult; use mysql_queries::queries::w2l::w2l_templates::list_w2l_templates::W2lTemplateRecordForList; -use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use r2d2_redis::{r2d2, RedisConnectionManager}; use redis_caching::redis_ttl_cache::RedisTtlCache; use reusable_types::server_environment::ServerEnvironment; @@ -22,7 +24,6 @@ use sqlx::MySqlPool; use url_config::third_party_url_redirector::ThirdPartyUrlRedirector; use users_component::utils::session_checker::SessionChecker; use users_component::utils::session_cookie_manager::SessionCookieManager; -use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList; /// State that is injected into every endpoint. pub struct ServerState { @@ -146,6 +147,11 @@ pub struct InMemoryCaches { /// This is approximately O(n^3) and recursively generates all super-category membership. pub tts_model_category_assignments: SingleItemTtlCache, + /// Generic inference queue length + /// The frontend will consult a distributed cache and use the monotonic DB time as a + /// vector clock. + pub inference_queue_length: SingleItemTtlCache, + /// TTS queue length /// The frontend will consult a distributed cache and use the monotonic DB time as a /// vector clock. @@ -168,12 +174,19 @@ pub struct StaticFeatureFlags { /// Used to bring the service back online slowly. pub global_429_pushback_filter_enabled: bool, + /// Disable the live `/v1/model_inference/queue_length` endpoint for all users and serve a static value instead. + pub disable_inference_queue_length_endpoint: bool, + /// Disable the live `/tts/queue_length` endpoint for all users and serve a static value instead. pub disable_tts_queue_length_endpoint: bool, /// Disable the live `/tts/list` endpoint for all users and serve a static value instead. pub disable_tts_model_list_endpoint: bool, + /// Tell the frontend client how fast to refresh their view of the pending inference count. + /// During an attack, we may want this to go extremely slow. + pub frontend_pending_inference_refresh_interval_millis: u64, + /// Tell the frontend client how fast to refresh their view of the pending TTS count. /// During an attack, we may want this to go extremely slow. pub frontend_pending_tts_refresh_interval_millis: u64,