mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-20 08:35:29 +00:00
inference status endpoints
This commit is contained in:
1 parent
5b97360724
commit
756c8e09ae
13 files changed
+568
-64
No files matched your search
@@ -17,12 +17,16 @@ CREATE TABLE generic_inference_jobs (
|
||||
|
||||
-- ========== INFERENCE DETAILS ==========
|
||||
|
||||
-- TODO: Rename this inference_category and add `maybe_model_type`
|
||||
-- Type of inference
|
||||
-- Examples (may not be up to date):
|
||||
-- * text_to_speech
|
||||
-- * voice_conversion
|
||||
inference_type VARCHAR(32) NOT NULL,
|
||||
|
||||
-- TODO: Add `maybe_model_type` here
|
||||
-- TODO: Move maybe_model_token here
|
||||
|
||||
-- Polymorphic arguments payload that depends on the type of inference job.
|
||||
-- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
|
||||
maybe_inference_args TEXT DEFAULT NULL,
|
||||
@@ -87,6 +91,7 @@ CREATE TABLE generic_inference_jobs (
|
||||
-- - (There will be future levels for paid Twitch and social FakeYou rewards.)
|
||||
priority_level SMALLINT UNSIGNED NOT NULL DEFAULT 0,
|
||||
|
||||
-- TODO: add is_for_storyteller_product
|
||||
is_from_premium_user BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_from_api_user BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
|
||||
@@ -47,7 +47,7 @@ impl <T, B> RouteBuilder<T, B>
|
||||
self.app
|
||||
}
|
||||
|
||||
/// Add an HTTP GET route.
|
||||
/// Add an HTTP GET route. This also adds the HEAD request for CORS.
|
||||
pub fn add_get<F, S, R>(mut self, path: &str, handler: F) -> Self
|
||||
where
|
||||
F: Handler<S, R>,
|
||||
@@ -63,7 +63,7 @@ impl <T, B> RouteBuilder<T, B>
|
||||
self
|
||||
}
|
||||
|
||||
/// Add an HTTP POST route.
|
||||
/// Add an HTTP POST route. This also adds the HEAD request for CORS.
|
||||
pub fn add_post<F, S, R>(mut self, path: &str, handler: F) -> Self
|
||||
where
|
||||
F: Handler<S, R>,
|
||||
|
||||
-59
@@ -1,59 +0,0 @@
|
||||
use anyhow::anyhow;
|
||||
use chrono::{DateTime, Utc};
|
||||
use errors::AnyhowResult;
|
||||
use log::warn;
|
||||
use sqlx::MySqlPool;
|
||||
use tokens::jobs::inference::InferenceJobToken;
|
||||
|
||||
pub struct GenericInferenceJobStatus {
|
||||
pub job_token: InferenceJobToken,
|
||||
|
||||
pub status: String,
|
||||
pub attempt_count: u16,
|
||||
|
||||
pub maybe_result_entity_type: Option<String>,
|
||||
pub maybe_result_entity_token: Option<String>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// Look up job status.
|
||||
/// Returns Ok(None) when the record cannot be found.
|
||||
pub async fn get_generic_inference_job_status(job_token: &InferenceJobToken, mysql_pool: &MySqlPool)
|
||||
-> AnyhowResult<Option<GenericInferenceJobStatus>>
|
||||
{
|
||||
let maybe_status = sqlx::query_as!(
|
||||
GenericInferenceJobStatus,
|
||||
r#"
|
||||
SELECT
|
||||
jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`,
|
||||
|
||||
jobs.status,
|
||||
jobs.attempt_count,
|
||||
jobs.on_success_result_entity_type as maybe_result_entity_type,
|
||||
jobs.on_success_result_entity_token as maybe_result_entity_token,
|
||||
|
||||
jobs.created_at,
|
||||
jobs.updated_at
|
||||
|
||||
FROM generic_inference_jobs as jobs
|
||||
|
||||
WHERE jobs.token = ?
|
||||
"#,
|
||||
job_token
|
||||
)
|
||||
.fetch_one(mysql_pool)
|
||||
.await;
|
||||
|
||||
match maybe_status {
|
||||
Ok(record) => Ok(Some(record)),
|
||||
Err(err) => match err {
|
||||
sqlx::Error::RowNotFound => Ok(None),
|
||||
_ => {
|
||||
warn!("error querying job record: {:?}", err);
|
||||
Err(anyhow!("error querying job record: {:?}", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
use anyhow::anyhow;
|
||||
use chrono::{DateTime, Utc};
|
||||
use errors::AnyhowResult;
|
||||
use log::warn;
|
||||
use sqlx::MySqlPool;
|
||||
use tokens::jobs::inference::InferenceJobToken;
|
||||
|
||||
pub struct GenericInferenceJobStatus {
|
||||
pub job_token: InferenceJobToken,
|
||||
|
||||
pub status: String,
|
||||
pub attempt_count: u16,
|
||||
|
||||
pub request_details: RequestDetails,
|
||||
pub maybe_result_details: Option<ResultDetails>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct RequestDetails {
|
||||
// TODO: Rename this to inference category!
|
||||
pub inference_type: String,
|
||||
pub maybe_model_token: Option<String>,
|
||||
|
||||
/// TTS input. In the future, perhaps voice conversion SST
|
||||
pub maybe_raw_inference_text: Option<String>,
|
||||
}
|
||||
|
||||
pub struct ResultDetails {
|
||||
pub entity_type: String,
|
||||
pub entity_token: String,
|
||||
}
|
||||
|
||||
/// Look up job status.
|
||||
/// Returns Ok(None) when the record cannot be found.
|
||||
pub async fn get_inference_job_status(job_token: &InferenceJobToken, mysql_pool: &MySqlPool)
|
||||
-> AnyhowResult<Option<GenericInferenceJobStatus>>
|
||||
{
|
||||
let maybe_status = sqlx::query_as!(
|
||||
RawGenericInferenceJobStatus,
|
||||
r#"
|
||||
SELECT
|
||||
jobs.token as `job_token: tokens::jobs::inference::InferenceJobToken`,
|
||||
|
||||
jobs.status,
|
||||
jobs.attempt_count,
|
||||
|
||||
jobs.inference_type,
|
||||
jobs.maybe_model_token,
|
||||
jobs.maybe_raw_inference_text,
|
||||
|
||||
jobs.on_success_result_entity_type as maybe_result_entity_type,
|
||||
jobs.on_success_result_entity_token as maybe_result_entity_token,
|
||||
|
||||
jobs.created_at,
|
||||
jobs.updated_at
|
||||
|
||||
FROM generic_inference_jobs as jobs
|
||||
|
||||
WHERE jobs.token = ?
|
||||
"#,
|
||||
job_token
|
||||
)
|
||||
.fetch_one(mysql_pool)
|
||||
.await;
|
||||
|
||||
let record = match maybe_status {
|
||||
Ok(record) => record,
|
||||
Err(err) => match err {
|
||||
sqlx::Error::RowNotFound => return Ok(None),
|
||||
_ => {
|
||||
warn!("error querying job record: {:?}", err);
|
||||
return Err(anyhow!("error querying job record: {:?}", err));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let maybe_result_details = record
|
||||
.maybe_result_entity_type
|
||||
.as_deref()
|
||||
.and_then(|entity_type| {
|
||||
record.maybe_result_entity_token
|
||||
.as_deref()
|
||||
.map(|entity_token| {
|
||||
ResultDetails {
|
||||
entity_type: entity_type.to_string(),
|
||||
entity_token: entity_token.to_string(),
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
Ok(Some(GenericInferenceJobStatus {
|
||||
job_token: record.job_token,
|
||||
status: record.status,
|
||||
attempt_count: record.attempt_count,
|
||||
request_details: RequestDetails {
|
||||
inference_type: record.inference_type,
|
||||
maybe_model_token: record.maybe_model_token,
|
||||
maybe_raw_inference_text: record.maybe_raw_inference_text,
|
||||
},
|
||||
maybe_result_details,
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
}))
|
||||
}
|
||||
|
||||
struct RawGenericInferenceJobStatus {
|
||||
pub job_token: InferenceJobToken,
|
||||
|
||||
pub status: String,
|
||||
pub attempt_count: u16,
|
||||
|
||||
pub inference_type: String,
|
||||
pub maybe_model_token: Option<String>,
|
||||
pub maybe_raw_inference_text: Option<String>,
|
||||
|
||||
pub maybe_result_entity_type: Option<String>,
|
||||
pub maybe_result_entity_token: Option<String>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
use chrono::NaiveDateTime;
|
||||
use crate::helpers::numeric_converters::try_i64_to_u64_or_min;
|
||||
use errors::AnyhowResult;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct InferenceQueueLengthResult {
|
||||
pub record_count: u64,
|
||||
pub present_time: NaiveDateTime,
|
||||
}
|
||||
|
||||
struct QueryResultInternal {
|
||||
record_count: i64,
|
||||
present_time: NaiveDateTime,
|
||||
}
|
||||
|
||||
pub async fn get_pending_inference_job_count(
|
||||
pool: &MySqlPool,
|
||||
) -> AnyhowResult<InferenceQueueLengthResult> {
|
||||
// NB (1): We query as a union since "started" jobs can get picked off mid-run and go stale
|
||||
// forever. We currently have no automated means of collecting those jobs.
|
||||
// NB (2): We query the server timestamp so we can cache the results.
|
||||
// The frontend can then monotonically adjust the count based on timestamp by ignoring
|
||||
// past times.
|
||||
let result : QueryResultInternal = sqlx::query_as!(
|
||||
QueryResultInternal,
|
||||
r#"
|
||||
SELECT
|
||||
COUNT(distinct token) as record_count,
|
||||
NOW() as present_time
|
||||
FROM
|
||||
(
|
||||
SELECT token
|
||||
FROM generic_inference_jobs
|
||||
WHERE status = "started"
|
||||
AND created_at > (CURDATE() - INTERVAL 5 MINUTE)
|
||||
UNION
|
||||
SELECT token
|
||||
FROM generic_inference_jobs
|
||||
WHERE status IN ("pending", "attempt_failed")
|
||||
) as t
|
||||
"#,
|
||||
)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
|
||||
Ok(InferenceQueueLengthResult {
|
||||
record_count: try_i64_to_u64_or_min(result.record_count),
|
||||
present_time: result.present_time,
|
||||
})
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod get_generic_inference_job_status;
|
||||
pub mod get_inference_job_status;
|
||||
pub mod get_pending_inference_job_count;
|
||||
pub mod insert_generic_inference_job;
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
use actix_http::Error;
|
||||
use actix_http::http::header;
|
||||
use actix_web::HttpResponseBuilder;
|
||||
use actix_web::cookie::Cookie;
|
||||
use actix_web::error::ResponseError;
|
||||
use actix_web::http::StatusCode;
|
||||
use actix_web::web::Path;
|
||||
use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage};
|
||||
use chrono::{DateTime, Utc};
|
||||
use crate::AnyhowResult;
|
||||
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
|
||||
use crate::server_state::ServerState;
|
||||
use log::{info, warn, log, error};
|
||||
use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status;
|
||||
use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status;
|
||||
use r2d2_redis::RedisConnectionManager;
|
||||
use r2d2_redis::r2d2::{Pool, PooledConnection};
|
||||
use r2d2_redis::redis::{Commands, RedisError, RedisResult};
|
||||
use redis_common::redis_keys::RedisKeys;
|
||||
use regex::Regex;
|
||||
use sqlx::MySqlPool;
|
||||
use sqlx::error::DatabaseError;
|
||||
use sqlx::error::Error::Database;
|
||||
use sqlx::mysql::MySqlDatabaseError;
|
||||
use std::borrow::BorrowMut;
|
||||
use std::fmt;
|
||||
use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
use tokens::jobs::inference::InferenceJobToken;
|
||||
|
||||
/// For the URL PathInfo
|
||||
#[derive(Deserialize)]
|
||||
pub struct GetInferenceJobStatusPathInfo {
|
||||
token: InferenceJobToken,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct InferenceJobStatusResponsePayload {
|
||||
pub job_token: InferenceJobToken,
|
||||
|
||||
pub request: RequestDetailsResponse,
|
||||
pub status: StatusDetailsResponse,
|
||||
pub maybe_result: Option<ResultDetailsResponse>,
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct RequestDetailsResponse {
|
||||
// TODO: Rename this inference_category (!!!) and add model_type.
|
||||
pub inference_category: String,
|
||||
pub maybe_model_type: Option<String>,
|
||||
pub maybe_model_token: Option<String>,
|
||||
|
||||
/// Title of the model
|
||||
pub title: String,
|
||||
|
||||
/// If the result was TTS, this is the raw inference text.
|
||||
pub maybe_raw_inference_text: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct StatusDetailsResponse {
|
||||
/// Primary status from the database (a state machine).
|
||||
pub status: String,
|
||||
|
||||
/// Extra, temporary status from Redis.
|
||||
/// This can denote inference progress, and the Python code can write to it.
|
||||
pub maybe_extra_status_description: Option<String>,
|
||||
|
||||
pub attempt_count: u8,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ResultDetailsResponse {
|
||||
pub entity_type: String,
|
||||
pub entity_token: String,
|
||||
|
||||
/// NB: This is only for audio- or video- type results.
|
||||
pub maybe_public_bucket_wav_media_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct GetInferenceJobStatusSuccessResponse {
|
||||
pub success: bool,
|
||||
pub state: InferenceJobStatusResponsePayload,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum GetInferenceJobStatusError {
|
||||
ServerError,
|
||||
NotFound,
|
||||
}
|
||||
|
||||
impl ResponseError for GetInferenceJobStatusError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
GetInferenceJobStatusError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
GetInferenceJobStatusError::NotFound => StatusCode::NOT_FOUND,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
let error_reason = match self {
|
||||
Self::ServerError => "server error".to_string(),
|
||||
Self::NotFound => "not found".to_string(),
|
||||
};
|
||||
|
||||
to_simple_json_error(&error_reason, self.status_code())
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl fmt::Display for GetInferenceJobStatusError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub async fn get_inference_job_status_handler(
|
||||
http_request: HttpRequest,
|
||||
path: Path<GetInferenceJobStatusPathInfo>,
|
||||
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, GetInferenceJobStatusError>
|
||||
{
|
||||
if path.token.as_str().trim() == "None" {
|
||||
// NB: A bunch of Python clients use our API and can fail in this manner.
|
||||
// This was a large traffic driver during the 2023-03-08 outage.
|
||||
return Err(GetInferenceJobStatusError::NotFound);
|
||||
}
|
||||
|
||||
// NB: Lookup failure is Err(RowNotFound).
|
||||
// NB: Since this is publicly exposed, we don't query sensitive data.
|
||||
let maybe_status = get_inference_job_status(&path.token, &server_state.mysql_pool).await;
|
||||
|
||||
let record = match maybe_status {
|
||||
Ok(Some(record)) => record,
|
||||
Ok(None) => return Err(GetInferenceJobStatusError::NotFound),
|
||||
Err(err) => {
|
||||
error!("tts job query error: {:?}", err);
|
||||
return Err(GetInferenceJobStatusError::ServerError);
|
||||
}
|
||||
};
|
||||
|
||||
let mut redis = server_state.redis_pool
|
||||
.get()
|
||||
.map_err(|e| {
|
||||
error!("redis error: {:?}", e);
|
||||
GetInferenceJobStatusError::ServerError
|
||||
})?;
|
||||
|
||||
let extra_status_key = RedisKeys::generic_inference_extra_status_info(path.token.as_str());
|
||||
let maybe_extra_status_description : Option<String> = match redis.get(&extra_status_key) {
|
||||
Ok(Some(status)) => {
|
||||
Some(status)
|
||||
},
|
||||
Ok(None) => None,
|
||||
Err(e) => {
|
||||
error!("redis error: {:?}", e);
|
||||
None // Fail open
|
||||
},
|
||||
};
|
||||
|
||||
let record_for_response = InferenceJobStatusResponsePayload {
|
||||
job_token: record.job_token,
|
||||
request: RequestDetailsResponse {
|
||||
inference_category: record.request_details.inference_type,
|
||||
maybe_model_type: None, // TODO: This will require a new field
|
||||
maybe_model_token: record.request_details.maybe_model_token,
|
||||
title: "AI Model".to_string(), // TODO: This will require polymorphic joins
|
||||
maybe_raw_inference_text: record.request_details.maybe_raw_inference_text,
|
||||
},
|
||||
status: StatusDetailsResponse {
|
||||
status: record.status,
|
||||
maybe_extra_status_description,
|
||||
attempt_count: record.attempt_count as u8,
|
||||
},
|
||||
maybe_result: record.maybe_result_details.map(|result_details| {
|
||||
ResultDetailsResponse {
|
||||
entity_type: result_details.entity_type,
|
||||
entity_token: result_details.entity_token,
|
||||
maybe_public_bucket_wav_media_path: None, // TODO: This will require a new field
|
||||
}
|
||||
}),
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
};
|
||||
|
||||
let response = GetInferenceJobStatusSuccessResponse {
|
||||
success: true,
|
||||
state: record_for_response,
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| {
|
||||
error!("error returning response: {:?}", e);
|
||||
GetInferenceJobStatusError::ServerError
|
||||
})?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
|
||||
use chrono::NaiveDateTime;
|
||||
use crate::ServerState;
|
||||
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
|
||||
use errors::AnyhowResult;
|
||||
use hyper::StatusCode;
|
||||
use log::{debug, error, info, warn};
|
||||
use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::get_pending_inference_job_count;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct GetPendingInferenceJobSuccessResponse {
|
||||
pub success: bool,
|
||||
pub pending_job_count: u64,
|
||||
pub cache_time: NaiveDateTime,
|
||||
|
||||
/// Tell the frontend client how fast to refresh their view of this list.
|
||||
/// During an attack, we may want this to go extremely slow.
|
||||
pub refresh_interval_millis: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum GetPendingInferenceJobCountError {
|
||||
ServerError,
|
||||
}
|
||||
|
||||
impl ResponseError for GetPendingInferenceJobCountError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
GetPendingInferenceJobCountError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
let error_reason = match self {
|
||||
GetPendingInferenceJobCountError::ServerError => "server error".to_string(),
|
||||
};
|
||||
|
||||
to_simple_json_error(&error_reason, self.status_code())
|
||||
}
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl fmt::Display for GetPendingInferenceJobCountError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_pending_inference_job_count_handler(
|
||||
http_request: HttpRequest,
|
||||
server_state: web::Data<Arc<ServerState>>
|
||||
) -> Result<HttpResponse, GetPendingInferenceJobCountError> {
|
||||
|
||||
if server_state.flags.disable_inference_queue_length_endpoint {
|
||||
// NB: Despite the cache being a powerful protector of the database (this is an expensive query),
|
||||
// if the cache goes stale during an outage, there is no protection. This feature flag lets us
|
||||
// shut off all traffic to the endpoint.
|
||||
return render_response_busy(GetPendingInferenceJobSuccessResponse {
|
||||
success: true,
|
||||
pending_job_count: 10_000,
|
||||
cache_time: NaiveDateTime::from_timestamp(0, 0),
|
||||
refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis,
|
||||
});
|
||||
}
|
||||
|
||||
let maybe_cached = server_state.caches.inference_queue_length.grab_copy_without_bump_if_unexpired()
|
||||
.map_err(|e| {
|
||||
error!("error consulting cache: {:?}", e);
|
||||
GetPendingInferenceJobCountError::ServerError
|
||||
})?;
|
||||
|
||||
let count_result = match maybe_cached {
|
||||
Some(cached) => {
|
||||
cached
|
||||
},
|
||||
None => {
|
||||
debug!("populating generic inference queue length from database");
|
||||
|
||||
let count_query_result = get_pending_inference_job_count(
|
||||
&server_state.mysql_pool
|
||||
).await;
|
||||
|
||||
match count_query_result {
|
||||
// If the database misbehaves (eg. DDoS), let's stop spamming it.
|
||||
// We'll attempt to read the old value from the cache and keep going.
|
||||
Err(err) => {
|
||||
warn!("error querying database / inserting into cache: {:?}", err);
|
||||
|
||||
let maybe_cached = server_state.caches.inference_queue_length.grab_even_expired_and_bump()
|
||||
.map_err(|err| {
|
||||
error!("error consulting cache (even expired): {:?}", err);
|
||||
GetPendingInferenceJobCountError::ServerError
|
||||
})?;
|
||||
|
||||
maybe_cached.ok_or_else(|| {
|
||||
error!("error querying database and subsequently reading cache: {:?}", err);
|
||||
GetPendingInferenceJobCountError::ServerError
|
||||
})?
|
||||
}
|
||||
|
||||
// Happy path...
|
||||
Ok(count_result) => {
|
||||
server_state.caches.inference_queue_length.store_copy(&count_result)
|
||||
.map_err(|e| {
|
||||
error!("error storing cache: {:?}", e);
|
||||
GetPendingInferenceJobCountError::ServerError
|
||||
})?;
|
||||
|
||||
count_result
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
render_response_ok(GetPendingInferenceJobSuccessResponse {
|
||||
success: true,
|
||||
pending_job_count: count_result.record_count,
|
||||
cache_time: count_result.present_time,
|
||||
refresh_interval_millis: server_state.flags.frontend_pending_inference_refresh_interval_millis,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn render_response_busy(response: GetPendingInferenceJobSuccessResponse) -> Result<HttpResponse, GetPendingInferenceJobCountError> {
|
||||
let body = render_response_payload(response)?;
|
||||
Ok(HttpResponse::TooManyRequests()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
|
||||
pub fn render_response_ok(response: GetPendingInferenceJobSuccessResponse) -> Result<HttpResponse, GetPendingInferenceJobCountError> {
|
||||
let body = render_response_payload(response)?;
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body))
|
||||
}
|
||||
|
||||
pub fn render_response_payload(response: GetPendingInferenceJobSuccessResponse) -> Result<String, GetPendingInferenceJobCountError> {
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| {
|
||||
error!("error returning response: {:?}", e);
|
||||
GetPendingInferenceJobCountError::ServerError
|
||||
})?;
|
||||
Ok(body)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod get_inference_job_status;
|
||||
pub mod get_pending_inference_job_count;
|
||||
@@ -4,6 +4,7 @@ pub mod comments;
|
||||
pub mod download_job;
|
||||
pub mod events;
|
||||
pub mod flags;
|
||||
pub mod inference_job;
|
||||
pub mod investor_demo;
|
||||
pub mod leaderboard;
|
||||
pub mod media_uploads;
|
||||
|
||||
@@ -296,6 +296,10 @@ async fn main() -> AnyhowResult<()> {
|
||||
easyenv::get_env_duration_seconds_or_default("LEADERBOARD_CACHE_TTL_SECONDS", Duration::from_secs(60))
|
||||
);
|
||||
|
||||
let inference_queue_length_cache = SingleItemTtlCache::create_with_duration(
|
||||
easyenv::get_env_duration_seconds_or_default("INFERENCE_QUEUE_LENGTH_CACHE_TTL_SECONDS", Duration::from_secs(30))
|
||||
);
|
||||
|
||||
// NB: This secret really isn't too important.
|
||||
// We can even rotate it without too much impact to users.
|
||||
let sort_key_crypto_secret =
|
||||
@@ -370,8 +374,10 @@ async fn main() -> AnyhowResult<()> {
|
||||
let service_feature_flags = StaticFeatureFlags {
|
||||
// Permanent (control plane / safety) flags
|
||||
global_429_pushback_filter_enabled: easyenv::get_env_bool_or_default("FF_GLOBAL_429_PUSHBACK_FILTER_ENABLED", false),
|
||||
disable_inference_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_INFERENCE_QUEUE_LENGTH_ENDPOINT", false),
|
||||
disable_tts_queue_length_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_QUEUE_LENGTH_ENDPOINT", false),
|
||||
disable_tts_model_list_endpoint: easyenv::get_env_bool_or_default("FF_DISABLE_TTS_MODEL_LIST_ENDPOINT", false),
|
||||
frontend_pending_inference_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_INFERENCE_REFRESH_INTERVAL_MILLIS", 15_000)?,
|
||||
frontend_pending_tts_refresh_interval_millis: easyenv::get_env_num("FF_FRONTEND_PENDING_TTS_REFRESH_INTERVAL_MILLIS", 15_000)?,
|
||||
troll_ban_user_percent: easyenv::get_env_num("FF_TROLL_BANNED_USER_PERCENT", 0)?,
|
||||
|
||||
@@ -438,6 +444,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
tts_queue_length: tts_queue_length_cache,
|
||||
tts_model_category_assignments: tts_model_category_assignments_cache,
|
||||
leaderboard: leaderboard_cache,
|
||||
inference_queue_length: inference_queue_length_cache,
|
||||
},
|
||||
twitch_oauth: TwitchOauth {
|
||||
secrets: TwitchOauthSecrets {
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::http_server::endpoints::download_job::get_generic_upload_job_status::
|
||||
use crate::http_server::endpoints::events::list_events::list_events_handler;
|
||||
use crate::http_server::endpoints::flags::design_refresh_flag::disable_design_refresh_flag_handler::disable_design_refresh_flag_handler;
|
||||
use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_refresh_flag_handler::enable_design_refresh_flag_handler;
|
||||
use crate::http_server::endpoints::inference_job::get_pending_inference_job_count::get_pending_inference_job_count_handler;
|
||||
use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler;
|
||||
use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler;
|
||||
use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler;
|
||||
@@ -110,6 +111,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t
|
||||
use users_component::default_routes::add_suggested_api_v1_account_creation_and_session_routes;
|
||||
use users_component::endpoints::edit_profile_handler::edit_profile_handler;
|
||||
use users_component::endpoints::get_profile_handler::get_profile_handler;
|
||||
use crate::http_server::endpoints::inference_job::get_inference_job_status::get_inference_job_status_handler;
|
||||
|
||||
pub fn add_routes<T, B> (app: App<T, B>) -> App<T, B>
|
||||
where
|
||||
@@ -149,6 +151,13 @@ pub fn add_routes<T, B> (app: App<T, B>) -> App<T, B>
|
||||
.add_post("/v1/comments/delete/{comment_token}", delete_comment_handler)
|
||||
.into_app();
|
||||
|
||||
// ==================== "Generic" Inference ====================
|
||||
|
||||
let mut app = RouteBuilder::from_app(app)
|
||||
.add_get("/v1/model_inference/job_status/{token}", get_inference_job_status_handler)
|
||||
.add_get("/v1/model_inference/queue_length", get_pending_inference_job_count_handler)
|
||||
.into_app();
|
||||
|
||||
// ==================== COMPONENTS ====================
|
||||
|
||||
app = add_suggested_api_v1_account_creation_and_session_routes(app); // /create_account, /session, /login, /logout
|
||||
|
||||
@@ -9,12 +9,14 @@ use crate::http_server::endpoints::tts::list_tts_models::TtsModelRecordForRespon
|
||||
use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter;
|
||||
use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus;
|
||||
use crate::util::encrypted_sort_id::SortKeyCrypto;
|
||||
use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList;
|
||||
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
|
||||
use mysql_queries::mediators::badge_granter::BadgeGranter;
|
||||
use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
|
||||
use mysql_queries::queries::generic_inference::web::get_pending_inference_job_count::InferenceQueueLengthResult;
|
||||
use mysql_queries::queries::model_categories::list_categories_query_builder::CategoryList;
|
||||
use mysql_queries::queries::tts::tts_inference_jobs::get_pending_tts_inference_job_count::TtsQueueLengthResult;
|
||||
use mysql_queries::queries::w2l::w2l_templates::list_w2l_templates::W2lTemplateRecordForList;
|
||||
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
|
||||
use r2d2_redis::{r2d2, RedisConnectionManager};
|
||||
use redis_caching::redis_ttl_cache::RedisTtlCache;
|
||||
use reusable_types::server_environment::ServerEnvironment;
|
||||
@@ -22,7 +24,6 @@ use sqlx::MySqlPool;
|
||||
use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
|
||||
use users_component::utils::session_checker::SessionChecker;
|
||||
use users_component::utils::session_cookie_manager::SessionCookieManager;
|
||||
use crate::util::troll_user_bans::troll_user_ban_list::TrollUserBanList;
|
||||
|
||||
/// State that is injected into every endpoint.
|
||||
pub struct ServerState {
|
||||
@@ -146,6 +147,11 @@ pub struct InMemoryCaches {
|
||||
/// This is approximately O(n^3) and recursively generates all super-category membership.
|
||||
pub tts_model_category_assignments: SingleItemTtlCache<ModelTokensByCategoryToken>,
|
||||
|
||||
/// Generic inference queue length
|
||||
/// The frontend will consult a distributed cache and use the monotonic DB time as a
|
||||
/// vector clock.
|
||||
pub inference_queue_length: SingleItemTtlCache<InferenceQueueLengthResult>,
|
||||
|
||||
/// TTS queue length
|
||||
/// The frontend will consult a distributed cache and use the monotonic DB time as a
|
||||
/// vector clock.
|
||||
@@ -168,12 +174,19 @@ pub struct StaticFeatureFlags {
|
||||
/// Used to bring the service back online slowly.
|
||||
pub global_429_pushback_filter_enabled: bool,
|
||||
|
||||
/// Disable the live `/v1/model_inference/queue_length` endpoint for all users and serve a static value instead.
|
||||
pub disable_inference_queue_length_endpoint: bool,
|
||||
|
||||
/// Disable the live `/tts/queue_length` endpoint for all users and serve a static value instead.
|
||||
pub disable_tts_queue_length_endpoint: bool,
|
||||
|
||||
/// Disable the live `/tts/list` endpoint for all users and serve a static value instead.
|
||||
pub disable_tts_model_list_endpoint: bool,
|
||||
|
||||
/// Tell the frontend client how fast to refresh their view of the pending inference count.
|
||||
/// During an attack, we may want this to go extremely slow.
|
||||
pub frontend_pending_inference_refresh_interval_millis: u64,
|
||||
|
||||
/// Tell the frontend client how fast to refresh their view of the pending TTS count.
|
||||
/// During an attack, we may want this to go extremely slow.
|
||||
pub frontend_pending_tts_refresh_interval_millis: u64,
|
||||
|
||||
Reference in new issue
Block a user