inference status endpoints

This commit is contained in:
Brandon Thomas committed 2023-04-08 18:49:59 -07:00
1 parent 5b97360724
commit 756c8e09ae
13 files changed
+568 -64

No files matched your search

@@ -17,12 +17,16 @@ CREATE TABLE generic_inference_jobs (
-- ========== INFERENCE DETAILS ==========
-- TODO: Rename this inference_category and add `maybe_model_type`
-- Type of inference
-- Examples (may not be up to date):
-- * text_to_speech
-- * voice_conversion
inference_type VARCHAR(32) NOT NULL,
-- TODO: Add `maybe_model_type` here
-- TODO: Move maybe_model_token here
-- Polymorphic arguments payload that depends on the type of inference job.
-- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
maybe_inference_args TEXT DEFAULT NULL,
@@ -87,6 +91,7 @@ CREATE TABLE generic_inference_jobs (
-- - (There will be future levels for paid Twitch and social FakeYou rewards.)
priority_level SMALLINT UNSIGNED NOT NULL DEFAULT 0,
-- TODO: add is_for_storyteller_product
is_from_premium_user BOOLEAN NOT NULL DEFAULT FALSE,
is_from_api_user BOOLEAN NOT NULL DEFAULT FALSE,
is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE,
@@ -47,7 +47,7 @@ impl <T, B> RouteBuilder<T, B>
self.app
}
/// Add an HTTP GET route.
/// Add an HTTP GET route. This also adds the HEAD request for CORS.
pub fn add_get<F, S, R>(mut self, path: &str, handler: F) -> Self
where
F: Handler<S, R>,
@@ -63,7 +63,7 @@ impl <T, B> RouteBuilder<T, B>
self
}
/// Add an HTTP POST route.
/// Add an HTTP POST route. This also adds the HEAD request for CORS.
pub fn add_post<F, S, R>(mut self, path: &str, handler: F) -> Self
where
F: Handler<S, R>,
@@ -1,59 +0,0 @@
use anyhow::anyhow;
use chrono::{DateTime, Utc};
use errors::AnyhowResult;
use log::warn;
use sqlx::MySqlPool;
use tokens::jobs::inference::InferenceJobToken;
pub struct GenericInferenceJobStatus {
pub job_token: InferenceJobToken,
pub status: String,
pub attempt_count: u16,
pub maybe_result_entity_type: Option<String>,
pub maybe_result_entity_token: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
/// Look up job status.
/// Returns Ok(None) when the record cannot be found.
pub async fn get_generic_inference_job_status(job_token: &InferenceJobToken, mysql_pool: &MySqlPool)
-> AnyhowResult<Option<GenericInferenceJobStatus>>
{
let maybe_status = sqlx::query_as!(
GenericInferenceJobStatus,
r#"
SELECT
jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`,
jobs.status,
jobs.attempt_count,
jobs.on_success_result_entity_type as maybe_result_entity_type,
jobs.on_success_result_entity_token as maybe_result_entity_token,
jobs.created_at,
jobs.updated_at
FROM generic_inference_jobs as jobs
WHERE jobs.token = ?
"#,
job_token
)
.fetch_one(mysql_pool)
.await;
match maybe_status {
Ok(record) => Ok(Some(record)),
Err(err) => match err {
sqlx::Error::RowNotFound => Ok(None),
_ => {
warn!("error querying job record: {:?}", err);
Err(anyhow!("error querying job record: {:?}", err))
}
}
}
}
@@ -0,0 +1,124 @@
use anyhow::anyhow;
use chrono::{DateTime, Utc};
use errors::AnyhowResult;
use log::warn;
use sqlx::MySqlPool;
use tokens::jobs::inference::InferenceJobToken;
pub struct GenericInferenceJobStatus {
pub job_token: InferenceJobToken,
pub status: String,
pub attempt_count: u16,
pub request_details: RequestDetails,
pub maybe_result_details: Option<ResultDetails>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
pub struct RequestDetails {
// TODO: Rename this to inference category!
pub inference_type: String,
pub maybe_model_token: Option<String>,
/// TTS input. In the future, perhaps voice conversion SST
pub maybe_raw_inference_text: Option<String>,
}
pub struct ResultDetails {
pub entity_type: String,
pub entity_token: String,
}
/// Look up job status.
/// Returns Ok(None) when the record cannot be found.
pub async fn get_inference_job_status(job_token: &InferenceJobToken, mysql_pool: &MySqlPool)
-> AnyhowResult<Option<GenericInferenceJobStatus>>
{
let maybe_status = sqlx::query_as!(
RawGenericInferenceJobStatus,
r#"
SELECT
jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`,
jobs.status,
jobs.attempt_count,
jobs.inference_type,
jobs.maybe_model_token,
jobs.maybe_raw_inference_text,
jobs.on_success_result_entity_type as maybe_result_entity_type,
jobs.on_success_result_entity_token as maybe_result_entity_token,
jobs.created_at,
jobs.updated_at
FROM generic_inference_jobs as jobs
WHERE jobs.token = ?
"#,
job_token
)
.fetch_one(mysql_pool)
.await;
let record = match maybe_status {
Ok(record) => record,
Err(err) => match err {
sqlx::Error::RowNotFound => return Ok(None),
_ => {
warn!("error querying job record: {:?}", err);
return Err(anyhow!("error querying job record: {:?}", err));
}
}
};
let maybe_result_details = record
.maybe_result_entity_type
.as_deref()
.and_then(|entity_type| {
record.maybe_result_entity_token
.as_deref()
.map(|entity_token| {
ResultDetails {
entity_type: entity_type.to_string(),
entity_token: entity_token.to_string(),
}
})
});
Ok(Some(GenericInferenceJobStatus {
job_token: record.job_token,
status: record.status,
attempt_count: record.attempt_count,
request_details: RequestDetails {
inference_type: record.inference_type,
maybe_model_token: record.maybe_model_token,
maybe_raw_inference_text: record.maybe_raw_inference_text,
},
maybe_result_details,
created_at: record.created_at,
updated_at: record.updated_at,
}))
}
struct RawGenericInferenceJobStatus {
pub job_token: InferenceJobToken,
pub status: String,
pub attempt_count: u16,
pub inference_type: String,
pub maybe_model_token: Option<String>,
pub maybe_raw_inference_text: Option<String>,
pub maybe_result_entity_type: Option<String>,
pub maybe_result_entity_token: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -0,0 +1,51 @@
use chrono::NaiveDateTime;
use crate::helpers::numeric_converters::try_i64_to_u64_or_min;
use errors::AnyhowResult;
use sqlx::MySqlPool;
#[derive(Clone)]
pub struct InferenceQueueLengthResult {
pub record_count: u64,
pub present_time: NaiveDateTime,
}
struct QueryResultInternal {
record_count: i64,
present_time: NaiveDateTime,
}
pub async fn get_pending_inference_job_count(
pool: &MySqlPool,
) -> AnyhowResult<InferenceQueueLengthResult> {
// NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale
// forever. We currently have no automated means of collecting those jobs.
// NB (2): We query the server timestamp so we can cache the results.
// The frontend can then monotonically adjust the count based on timestamp by ignoring
// past times.
let result : QueryResultInternal = sqlx::query_as!(
QueryResultInternal,
r#"
SELECT
COUNT(distinct token) as record_count,
NOW() as present_time
FROM
(
SELECT token
FROM generic_inference_jobs
WHERE status = "started"
AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
UNION
SELECT token
FROM generic_inference_jobs
WHERE status IN ("pending", "attempt_failed")
) as t
"#,
)
.fetch_one(pool)
.await?;
Ok(InferenceQueueLengthResult {
record_count: try_i64_to_u64_or_min(result.record_count),
present_time: result.present_time,
})
}
@@ -1,2 +1,3 @@
pub mod get_generic_inference_job_status;
pub mod get_inference_job_status;
pub mod get_pending_inference_job_count;
pub mod insert_generic_inference_job;
@@ -0,0 +1,204 @@
use actix_http::Error;
use actix_http::http::header;
use actix_web::HttpResponseBuilder;
use actix_web::cookie::Cookie;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::Path;
use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage};
use chrono::{DateTime, Utc};
use crate::AnyhowResult;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
use log::{info, warn, log, error};
use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status;
use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status;
use r2d2_redis::RedisConnectionManager;
use r2d2_redis::r2d2::{Pool, PooledConnection};
use r2d2_redis::redis::{Commands, RedisError, RedisResult};
use redis_common::redis_keys::RedisKeys;
use regex::Regex;
use sqlx::MySqlPool;
use sqlx::error::DatabaseError;
use sqlx::error::Error::Database;
use sqlx::mysql::MySqlDatabaseError;
use std::borrow::BorrowMut;
use std::fmt;
use std::ops::Deref;
use std::sync::Arc;
use tokens::jobs::inference::InferenceJobToken;
/// For the URL PathInfo
#[derive(Deserialize)]
pub struct GetInferenceJobStatusPathInfo {
token: InferenceJobToken,
}
#[derive(Serialize)]
pub struct InferenceJobStatusResponsePayload {
pub job_token: InferenceJobToken,
pub request: RequestDetailsResponse,
pub status: StatusDetailsResponse,
pub maybe_result: Option<ResultDetailsResponse>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Serialize)]
pub struct RequestDetailsResponse {
// TODO: Rename this inference_category (!!!) and add model_type.
pub inference_category: String,
pub maybe_model_type: Option<String>,
pub maybe_model_token: Option<String>,
/// Title of the model
pub title: String,
/// If the result was TTS, this is the raw inference text.
pub maybe_raw_inference_text: Option<String>,
}
#[derive(Serialize)]
pub struct StatusDetailsResponse {
/// Primary status from the database (a state machine).
pub status: String,
/// Extra, temporary status from Redis.
/// This can denote inference progress, and the Python code can write to it.
pub maybe_extra_status_description: Option<String>,
pub attempt_count: u8,
}
#[derive(Serialize)]
pub struct ResultDetailsResponse {
pub entity_type: String,
pub entity_token: String,
/// NB: This is only for audio- or video- type results.
pub maybe_public_bucket_wav_media_path: Option<String>,
}
#[derive(Serialize)]
pub struct GetInferenceJobStatusSuccessResponse {
pub success: bool,
pub state: InferenceJobStatusResponsePayload,
}
#[derive(Debug)]
pub enum GetInferenceJobStatusError {
ServerError,
NotFound,
}
impl ResponseError for GetInferenceJobStatusError {
fn status_code(&self) -> StatusCode {
match *self {
GetInferenceJobStatusError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
GetInferenceJobStatusError::NotFound => StatusCode::NOT_FOUND,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
Self::ServerError => "server error".to_string(),
Self::NotFound => "not found".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for GetInferenceJobStatusError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
pub async fn get_inference_job_status_handler(
http_request: HttpRequest,
path: Path<GetInferenceJobStatusPathInfo>,
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, GetInferenceJobStatusError>
{
if path.token.as_str().trim() == "None" {
// NB: A bunch of Python clients use our API and can fail in this manner.
// This was a large traffic driver during the 2023-03-08 outage.
return Err(GetInferenceJobStatusError::NotFound);
}
// NB: Lookup failure is Err(RowNotFound).
// NB: Since this is publicly exposed, we don't query sensitive data.
let maybe_status = get_inference_job_status(&path.token, &server_state.mysql_pool).await;
let record = match maybe_status {
Ok(Some(record)) => record,
Ok(None) => return Err(GetInferenceJobStatusError::NotFound),
Err(err) => {
error!("tts job query error: {:?}", err);
return Err(GetInferenceJobStatusError::ServerError);
}
};
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
error!("redis error: {:?}", e);
GetInferenceJobStatusError::ServerError
})?;
let extra_status_key = RedisKeys::generic_inference_extra_status_info(path.token.as_str());
let maybe_extra_status_description : Option<String> = match redis.get(&extra_status_key) {
Ok(Some(status)) => {
Some(status)
},
Ok(None) => None,
Err(e) => {
error!("redis error: {:?}", e);
None // Fail open
},
};
let record_for_response = InferenceJobStatusResponsePayload {
job_token: record.job_token,
request: RequestDetailsResponse {
inference_category: record.request_details.inference_type,
maybe_model_type: None, // TODO: This will require a new field
maybe_model_token: record.request_details.maybe_model_token,
title: "AI Model".to_string(), // TODO: This will require polymorphic joins
maybe_raw_inference_text: record.request_details.maybe_raw_inference_text,
},
status: StatusDetailsResponse {
status: record.status,
maybe_extra_status_description,
attempt_count: record.attempt_count as u8,
},
maybe_result: record.maybe_result_details.map(|result_details| {
ResultDetailsResponse {
entity_type: result_details.entity_type,
entity_token: result_details.entity_token,
maybe_public_bucket_wav_media_path: None, // TODO: This will require a new field
}
}),
created_at: record.created_at,
updated_at: record.updated_at,
};
let response = GetInferenceJobStatusSuccessResponse {
success: true,
state: record_for_response,
};
let body = serde_json::to_string(&response)
.map_err(|e| {
error!("error returning response: {:?}", e);
GetInferenceJobStatusError::ServerError
})?;
Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body))
}
@@ -0,0 +1,146 @@
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
use chrono::NaiveDateTime;
use crate::ServerState;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use errors::AnyhowResult;
use hyper::StatusCode;
use log::{debug, error, info, warn};
use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::get_pending_inference_job_count;
use std::fmt;
use std::sync::Arc;
#[derive(Serialize)]
pub struct GetPendingInferenceJobSuccessResponse {
pub success: bool,
pub pending_job_count: u64,
pub cache_time: NaiveDateTime,
/// Tell the frontend client how fast to refresh their view of this list.
/// During an attack, we may want this to go extremely slow.
pub refresh_interval_millis: u64,
}
#[derive(Debug)]
pub enum GetPendingInferenceJobCountError {
ServerError,
}
impl ResponseError for GetPendingInferenceJobCountError {
fn status_code(&self) -> StatusCode {
match *self {
GetPendingInferenceJobCountError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
GetPendingInferenceJobCountError::ServerError => "server error".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for GetPendingInferenceJobCountError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
pub async fn get_pending_inference_job_count_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>
) -> Result<HttpResponse, GetPendingInferenceJobCountError> {
if server_state.flags.disable_inference_queue_length_endpoint {
// NB: Despite the cache being a powerful protector of the database (this is an expensive query),
// if the cache goes stale during an outage, there is no protection. This feature flag lets us
// shut off all traffic to the endpoint.
return render_response_busy(GetPendingInferenceJobSuccessResponse {
success: true,
pending_job_count: 10_000,
cache_time: NaiveDateTime::from_timestamp(0, 0),
refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis,
});
}
let maybe_cached = server_state.caches.inference_queue_length.grab_copy_without_bump_if_unexpired()
.map_err(|e| {
error!("error consulting cache: {:?}", e);
GetPendingInferenceJobCountError::ServerError
})?;
let count_result = match maybe_cached {
Some(cached) => {
cached
},
None => {
debug!("populating generic inference queue length from database");
let count_query_result = get_pending_inference_job_count(
&server_state.mysql_pool
).await;
match count_query_result {
// If the database misbehaves (eg. DDoS), let's stop spamming it.
// We'll attempt to read the old value from the cache and keep going.
Err(err) => {
warn!("error querying database / inserting into cache: {:?}", err);
let maybe_cached = server_state.caches.inference_queue_length.grab_even_expired_and_bump()
.map_err(|err| {
error!("error consulting cache (even expired): {:?}", err);
GetPendingInferenceJobCountError::ServerError
})?;
maybe_cached.ok_or_else(|| {
error!("error querying database and subsequently reading cache: {:?}", err);
GetPendingInferenceJobCountError::ServerError
})?
}
// Happy path...
Ok(count_result) => {
server_state.caches.inference_queue_length.store_copy(&count_result)
.map_err(|e| {
error!("error storing cache: {:?}", e);
GetPendingInferenceJobCountError::ServerError
})?;
count_result
}
}
},
};
render_response_ok(GetPendingInferenceJobSuccessResponse {
success: true,
pending_job_count: count_result.record_count,
cache_time: count_result.present_time,
refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis,
})
}
pub fn render_response_busy(response: GetPendingInferenceJobSuccessResponse) -> Result<HttpResponse, GetPendingInferenceJobCountError> {
let body = render_response_payload(response)?;
Ok(HttpResponse::TooManyRequests()
.content_type("application/json")
.body(body))
}
pub fn render_response_ok(response: GetPendingInferenceJobSuccessResponse) -> Result<HttpResponse, GetPendingInferenceJobCountError> {
let body = render_response_payload(response)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body))
}
pub fn render_response_payload(response: GetPendingInferenceJobSuccessResponse) -> Result<String, GetPendingInferenceJobCountError> {
let body = serde_json::to_string(&response)
.map_err(|e| {
error!("error returning response: {:?}", e);
GetPendingInferenceJobCountError::ServerError
})?;
Ok(body)
}
@@ -0,0 +1,2 @@
pub mod get_inference_job_status;
pub mod get_pending_inference_job_count;
@@ -4,6 +4,7 @@ pub mod comments;
pub mod download_job;
pub mod events;
pub mod flags;
pub mod inference_job;
pub mod investor_demo;
pub mod leaderboard;
pub mod media_uploads;
@@ -296,6 +296,10 @@ async fn main() -> AnyhowResult<()> {
easyenv::get_env_duration_seconds_or_default("LEADERBOARD_CACHE_TTL_SECONDS", Duration::from_secs(60))
);
let inference_queue_length_cache = SingleItemTtlCache::create_with_duration(
easyenv::get_env_duration_seconds_or_default("INFERENCE_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30))
);
// NB: This secret really isn't too important.
// We can even rotate it without too much impact to users.
let sort_key_crypto_secret =
@@ -370,8 +374,10 @@ async fn main() -> AnyhowResult<()> {
let service_feature_flags = StaticFeatureFlags {
// Permanent (control plane / safety) flags
global_429_pushback_filter_enabled: easyenv::get_env_bool_or_default("FF_GLOBAL_429_PUSHBACK_FILTER_ENABLED", false),
disable_inference_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_INFERENCE_QUEUE_LENGTH_ENDPOINT", false),
disable_tts_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_QUEUE_LENGTH_ENDPOINT", false),
disable_tts_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_MODEL_LIST_ENDPOINT", false),
frontend_pending_inference_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_INFERENCE_REFRESH_INTERVAL_MILLIS", 15_000)?,
frontend_pending_tts_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_TTS_REFRESH_INTERVAL_MILLIS", 15_000)?,
troll_ban_user_percent: easyenv::get_env_num("FF_TROLL_BANNED_USER_PERCENT", 0)?,
@@ -438,6 +444,7 @@ async fn main() -> AnyhowResult<()> {
tts_queue_length: tts_queue_length_cache,
tts_model_category_assignments: tts_model_category_assignments_cache,
leaderboard: leaderboard_cache,
inference_queue_length: inference_queue_length_cache,
},
twitch_oauth: TwitchOauth {
secrets: TwitchOauthSecrets {
@@ -23,6 +23,7 @@ use crate::http_server::endpoints::download_job::get_generic_upload_job_status::
use crate::http_server::endpoints::events::list_events::list_events_handler;
use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler;
use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler;
use crate::http_server::endpoints::inference_job::get_pending_inference_job_count::get_pending_inference_job_count_handler;
use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler;
use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler;
use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler;
@@ -110,6 +111,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t
use users_component::default_routes::add_suggested_api_v1_account_creation_and_session_routes;
use users_component::endpoints::edit_profile_handler::edit_profile_handler;
use users_component::endpoints::get_profile_handler::get_profile_handler;
use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler;
pub fn add_routes<T, B> (app: App<T, B>) -> App<T, B>
where
@@ -149,6 +151,13 @@ pub fn add_routes<T, B> (app: App<T, B>) -> App<T, B>
.add_post("/v1/comments/delete/{comment_token}", delete_comment_handler)
.into_app();
// ==================== "Generic" Inference ====================
let mut app = RouteBuilder::from_app(app)
.add_get("/v1/model_inference/job_status/{token}", get_inference_job_status_handler)
.add_get("/v1/model_inference/queue_length", get_pending_inference_job_count_handler)
.into_app();
// ==================== COMPONENTS ====================
app = add_suggested_api_v1_account_creation_and_session_routes(app); // /create_account, /session, /login, /logout
@@ -9,12 +9,14 @@ use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForRespon
use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter;
use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus;
use crate::util::encrypted_sort_id::SortKeyCrypto;
use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList;
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
use mysql_queries::mediators::badge_granter::BadgeGranter;
use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult;
use mysql_queries::queries::model_categories::list_categories_query_builder::CategoryList;
use mysql_queries::queries::tts::tts_inference_jobs::get_pending_tts_inference_job_count::TtsQueueLengthResult;
use mysql_queries::queries::w2l::w2l_templates::list_w2l_templates::W2lTemplateRecordForList;
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
use r2d2_redis::{r2d2, RedisConnectionManager};
use redis_caching::redis_ttl_cache::RedisTtlCache;
use reusable_types::server_environment::ServerEnvironment;
@@ -22,7 +24,6 @@ use sqlx::MySqlPool;
use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
use users_component::utils::session_checker::SessionChecker;
use users_component::utils::session_cookie_manager::SessionCookieManager;
use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList;
/// State that is injected into every endpoint.
pub struct ServerState {
@@ -146,6 +147,11 @@ pub struct InMemoryCaches {
/// This is approximately O(n^3) and recursively generates all super-category membership.
pub tts_model_category_assignments: SingleItemTtlCache<ModelTokensByCategoryToken>,
/// Generic inference queue length
/// The frontend will consult a distributed cache and use the monotonic DB time as a
/// vector clock.
pub inference_queue_length: SingleItemTtlCache<InferenceQueueLengthResult>,
/// TTS queue length
/// The frontend will consult a distributed cache and use the monotonic DB time as a
/// vector clock.
@@ -168,12 +174,19 @@ pub struct StaticFeatureFlags {
/// Used to bring the service back online slowly.
pub global_429_pushback_filter_enabled: bool,
/// Disable the live `/v1/model_inference/queue_length` endpoint for all users and serve a static value instead.
pub disable_inference_queue_length_endpoint: bool,
/// Disable the live `/tts/queue_length` endpoint for all users and serve a static value instead.
pub disable_tts_queue_length_endpoint: bool,
/// Disable the live `/tts/list` endpoint for all users and serve a static value instead.
pub disable_tts_model_list_endpoint: bool,
/// Tell the frontend client how fast to refresh their view of the pending inference count.
/// During an attack, we may want this to go extremely slow.
pub frontend_pending_inference_refresh_interval_millis: u64,
/// Tell the frontend client how fast to refresh their view of the pending TTS count.
/// During an attack, we may want this to go extremely slow.
pub frontend_pending_tts_refresh_interval_millis: u64,