diff --git a/crates/schema/database/mysql_queries/src/queries/model_weights/list/list_featured_weights.rs b/crates/schema/database/mysql_queries/src/queries/model_weights/list/list_featured_weights.rs deleted file mode 100644 index f8c08ac3d0..0000000000 --- a/crates/schema/database/mysql_queries/src/queries/model_weights/list/list_featured_weights.rs +++ /dev/null @@ -1,472 +0,0 @@ -use std::collections::HashSet; -use chrono::{DateTime, Utc}; -use sqlx::{FromRow, MySql, MySqlPool, QueryBuilder, Row}; -use sqlx::mysql::MySqlRow; - -use enums::by_table::model_weights::{ - weights_category::WeightsCategory, - weights_types::WeightsType, -}; -use enums::common::view_as::ViewAs; -use enums::traits::mysql_from_row::MySqlFromRow as _; -use enums::common::visibility::Visibility; -use errors::AnyhowResult; -use tokens::tokens::model_weights::ModelWeightToken; -use tokens::tokens::users::UserToken; - -pub struct FeaturedWeightsPage { - pub records: Vec, - - pub sort_ascending: bool, - - /// ID of the first record in the result set. - pub first_id: Option, - - /// ID of the last record in the result set. - pub last_id: Option, -} - -#[derive(Serialize)] -pub struct FeaturedWeight { - pub token: ModelWeightToken, - - pub weights_type: WeightsType, - pub weights_category: WeightsCategory, - - pub title: String, - - // TODO(bt,2023-12-24): These aren't really appropriate for a list endpoint. - // Hopefully we don't break the frontend by omitting these. - //pub maybe_description_markdown: String, - //pub maybe_description_rendered_html: String, - - pub maybe_ietf_language_tag: Option, - pub maybe_ietf_primary_language_subtag: Option, - - pub maybe_creator_user_token: Option, - pub maybe_creator_username: Option, - pub maybe_creator_display_name: Option, - pub maybe_creator_email_gravatar_hash: Option, - - pub creator_ip_address: String, - pub creator_set_visibility: Visibility, - - pub maybe_last_update_user_token: Option, - - pub original_download_url: Option, - pub original_filename: Option, - - pub file_size_bytes: i32, - pub file_checksum_sha2: String, - - pub public_bucket_hash: String, - pub maybe_public_bucket_prefix: Option, - pub maybe_public_bucket_extension: Option, - - pub maybe_cover_image_public_bucket_hash: Option, - pub maybe_cover_image_public_bucket_prefix: Option, - pub maybe_cover_image_public_bucket_extension: Option, - - pub maybe_ratings_positive_count: Option, - pub maybe_ratings_negative_count: Option, - pub maybe_bookmark_count: Option, - pub cached_usage_count: u64, - - pub version: i32, - - pub created_at: DateTime, - pub updated_at: DateTime, - - pub user_deleted_at: Option>, - pub mod_deleted_at: Option>, -} - -pub struct ListFeaturedWeightsArgs<'a> { - pub limit: usize, - pub maybe_offset: Option, - - pub cursor_is_reversed: bool, - pub sort_ascending: bool, - - pub view_as: ViewAs, - - pub maybe_scoped_weight_types: Option<&'a HashSet>, - pub maybe_scoped_weight_categories: Option<&'a HashSet>, - - pub mysql_pool: &'a MySqlPool, -} - -pub async fn list_featured_weights(args: ListFeaturedWeightsArgs<'_>) -> AnyhowResult { - // TODO(bt,2024-05-05): Why were we doing this, again? - // let count_fields = select_total_count_field(); - // let mut count_query_builder = query_builder( - // args.limit, - // args.maybe_offset, - // args.cursor_is_reversed, - // args.sort_ascending, - // count_fields.as_str(), - // args.view_as, - // args.maybe_scoped_weight_type, - // args.maybe_scoped_weight_category, - // ); - // let row_count_query = count_query_builder.build_query_scalar::(); - // let row_count_result = row_count_query.fetch_one(args.mysql_pool).await?; - - /// Now fetch the actual results with all the fields - let result_fields = select_result_fields(); - let mut query = query_builder( - args.limit, - args.maybe_offset, - args.cursor_is_reversed, - args.sort_ascending, - result_fields.as_str(), - args.view_as, - args.maybe_scoped_weight_types, - args.maybe_scoped_weight_categories, - ); - - let query = query.build_query_as::(); - let results = query.fetch_all(args.mysql_pool).await?; - - let first_id = results.first() - .map(|raw_result| raw_result.id); - - let last_id = results.last() - .map(|raw_result| raw_result.id); - - let weights_records: Vec = map_to_weights(results).await; - - Ok(FeaturedWeightsPage { - records: weights_records, - sort_ascending: args.sort_ascending, - first_id, - last_id, - }) -} - -fn select_total_count_field() -> String { - r#" - COUNT(mw.id) AS total_count - "# - .to_string() -} - -fn select_result_fields() -> String { - r#" - mw.id, - mw.token, - mw.title, - mw.weights_type, - mw.weights_category, - mw.maybe_ietf_language_tag, - mw.maybe_ietf_primary_language_subtag, - u.token as maybe_creator_user_token, - u.username as maybe_creator_username, - u.display_name as maybe_creator_display_name, - u.email_gravatar_hash as maybe_creator_email_gravatar_hash, - mw.creator_ip_address, - mw.creator_set_visibility, - mw.maybe_last_update_user_token, - mw.original_download_url, - mw.original_filename, - mw.file_size_bytes, - mw.file_checksum_sha2, - - mw.public_bucket_hash, - mw.maybe_public_bucket_prefix, - mw.maybe_public_bucket_extension, - - cover_image.public_bucket_directory_hash as maybe_cover_image_public_bucket_hash, - cover_image.maybe_public_bucket_prefix as maybe_cover_image_public_bucket_prefix, - cover_image.maybe_public_bucket_extension as maybe_cover_image_public_bucket_extension, - - entity_stats.ratings_positive_count as maybe_ratings_positive_count, - entity_stats.ratings_negative_count as maybe_ratings_negative_count, - entity_stats.bookmark_count as maybe_bookmark_count, - mw.cached_usage_count, - - mw.version, - mw.created_at, - mw.updated_at, - mw.user_deleted_at, - mw.mod_deleted_at - "#.to_string() -} - -fn query_builder<'a>( - _limit: usize, // TODO(bt,2024-05-16): This seems wrong. Why is this unused? - maybe_offset: Option, - cursor_is_reversed: bool, - sort_ascending: bool, - select_fields: &'a str, - view_as: ViewAs, - maybe_scoped_weight_types: Option<&'a HashSet>, - maybe_scoped_weight_categories: Option<&'a HashSet>, -) -> QueryBuilder<'a, MySql> { - - // NB: Query cannot be statically checked by sqlx - let mut query_builder: QueryBuilder = QueryBuilder::new( - format!(r#" -SELECT - {select_fields} -FROM model_weights as mw - -JOIN featured_items - ON featured_items.entity_type = "model_weight" - AND featured_items.deleted_at IS NULL - AND featured_items.entity_token = mw.token - -LEFT OUTER JOIN users as u - ON u.token = mw.creator_user_token -LEFT OUTER JOIN media_files as cover_image - ON cover_image.token = mw.maybe_cover_image_media_file_token -LEFT OUTER JOIN entity_stats - ON entity_stats.entity_type = "model_weights" - AND entity_stats.entity_token = mw.token - "# - )); - - let mut first_predicate_added = false; - - match view_as { - ViewAs::Author | ViewAs::Moderator => { - if !first_predicate_added { - query_builder.push(" WHERE "); - first_predicate_added = true; - } else { - query_builder.push(" AND "); - } - // NB(bt): Actually, mods don't want to see deleted models. We'll improve the moderator UI later. - query_builder.push(" mw.user_deleted_at IS NULL AND mw.mod_deleted_at IS NULL "); - } - ViewAs::AnotherUser => { - if !first_predicate_added { - query_builder.push(" WHERE "); - first_predicate_added = true; - } else { - query_builder.push(" AND "); - } - query_builder.push(" mw.user_deleted_at IS NULL AND mw.mod_deleted_at IS NULL "); - query_builder.push(" AND mw.creator_set_visibility = "); - query_builder.push_bind(Visibility::Public.to_str()); - } - } - - if let Some(weight_types) = maybe_scoped_weight_types { - // NB: `WHERE IN` comma separated syntax will be wrong if list has zero length - // We'll skip the predicate if the list isn't empty. - if !weight_types.is_empty() { - if !first_predicate_added { - query_builder.push(" WHERE "); - first_predicate_added = true; - } else { - query_builder.push(" AND "); - } - query_builder.push(" mw.weights_type IN ( "); - let mut separated = query_builder.separated(", "); - for weight_type in weight_types.iter() { - separated.push_bind(weight_type.to_str()); - } - separated.push_unseparated(") "); - } - } - - if let Some(weight_categories) = maybe_scoped_weight_categories { - // NB: `WHERE IN` comma separated syntax will be wrong if list has zero length - // We'll skip the predicate if the list isn't empty. - if !weight_categories.is_empty() { - if !first_predicate_added { - query_builder.push(" WHERE "); - first_predicate_added = true; - } else { - query_builder.push(" AND "); - } - query_builder.push(" mw.weights_category IN ( "); - let mut separated = query_builder.separated(", "); - for weights_category in weight_categories.iter() { - separated.push_bind(weights_category.to_str()); - } - separated.push_unseparated(") "); - } - } - - if let Some(offset) = maybe_offset { - if !first_predicate_added { - query_builder.push(" WHERE "); - first_predicate_added = true; - } else { - query_builder.push(" AND "); - } - - let mut sort_ascending = sort_ascending; - - if sort_ascending { - if cursor_is_reversed { - // NB: We're searching backwards. - query_builder.push(" m.id < "); - sort_ascending = !sort_ascending; - } else { - query_builder.push(" m.id > "); - } - } else { - if cursor_is_reversed { - // NB: We're searching backwards. - query_builder.push(" m.id > "); - sort_ascending = !sort_ascending; - } else { - query_builder.push(" m.id < "); - } - } - query_builder.push_bind(offset as i64); - } - - query_builder -} - -async fn map_to_weights(dataset:Vec) -> Vec { - let weights: Vec = dataset - .into_iter() - .map(|weight: RawWeightJoinUser| { - FeaturedWeight { - token: weight.token, - title: weight.title, - weights_type: weight.weights_type, - weights_category: weight.weights_category, - - maybe_ietf_language_tag: weight.maybe_ietf_language_tag, - maybe_ietf_primary_language_subtag: weight.maybe_ietf_primary_language_subtag, - - maybe_creator_user_token: weight.maybe_creator_user_token, - maybe_creator_username: weight.maybe_creator_username, - maybe_creator_display_name: weight.maybe_creator_display_name, - maybe_creator_email_gravatar_hash: weight.maybe_creator_email_gravatar_hash, - - creator_ip_address: weight.creator_ip_address, - creator_set_visibility: weight.creator_set_visibility, - - maybe_last_update_user_token: weight.maybe_last_update_user_token, - original_download_url: weight.original_download_url, - original_filename: weight.original_filename, - file_size_bytes: weight.file_size_bytes, - file_checksum_sha2: weight.file_checksum_sha2, - public_bucket_hash: weight.public_bucket_hash, - maybe_public_bucket_prefix: weight.maybe_public_bucket_prefix, - maybe_public_bucket_extension: weight.maybe_public_bucket_extension, - - maybe_cover_image_public_bucket_hash: weight.maybe_cover_image_public_bucket_hash, - maybe_cover_image_public_bucket_prefix: weight.maybe_cover_image_public_bucket_prefix, - maybe_cover_image_public_bucket_extension: weight.maybe_cover_image_public_bucket_extension, - - maybe_ratings_positive_count: weight.maybe_ratings_positive_count, - maybe_ratings_negative_count: weight.maybe_ratings_negative_count, - maybe_bookmark_count: weight.maybe_bookmark_count, - cached_usage_count: weight.cached_usage_count, - - version: weight.version, - created_at: weight.created_at, - updated_at: weight.updated_at, - user_deleted_at: weight.user_deleted_at, - mod_deleted_at: weight.mod_deleted_at, - } - }).collect(); - weights -} - -struct RawWeightJoinUser { - id: i64, - token: ModelWeightToken, - - weights_type: WeightsType, - weights_category: WeightsCategory, - - title: String, - - maybe_ietf_language_tag: Option, - maybe_ietf_primary_language_subtag: Option, - - // TODO(bt,2023-12-24): These aren't really appropriate for a list endpoint. - // Hopefully we don't break the frontend by omitting these. - //pub description_markdown: String, - //pub description_rendered_html: String, - - maybe_creator_user_token: Option, - maybe_creator_username: Option, - maybe_creator_display_name: Option, - maybe_creator_email_gravatar_hash: Option, - - creator_ip_address: String, - creator_set_visibility: Visibility, - - maybe_last_update_user_token: Option, - - original_download_url: Option, - original_filename: Option, - - file_size_bytes: i32, - file_checksum_sha2: String, - - public_bucket_hash: String, - maybe_public_bucket_prefix: Option, - maybe_public_bucket_extension: Option, - cached_usage_count: u64, - - maybe_cover_image_public_bucket_hash: Option, - maybe_cover_image_public_bucket_prefix: Option, - maybe_cover_image_public_bucket_extension: Option, - - maybe_ratings_positive_count: Option, - maybe_ratings_negative_count: Option, - maybe_bookmark_count: Option, - - version: i32, - - created_at: DateTime, - updated_at: DateTime, - - user_deleted_at: Option>, - mod_deleted_at: Option>, -} - -impl FromRow<'_, MySqlRow> for RawWeightJoinUser { - fn from_row(row: &MySqlRow) -> Result { - Ok(Self { - id: row.try_get("id")?, - token: row.try_get("token")?, - weights_type: row.try_get("weights_type")?, - weights_category: row.try_get("weights_category")?, - title: row.try_get("title")?, - maybe_ietf_language_tag: row.try_get("maybe_ietf_language_tag")?, - maybe_ietf_primary_language_subtag: row.try_get("maybe_ietf_primary_language_subtag")?, - // TODO(bt,2023-12-24): These aren't really appropriate for a list endpoint. - // Hopefully we don't break the frontend by omitting these. - //maybe_description_markdown: row.try_get("maybe_description_markdown")?, - //maybe_description_rendered_html: row.try_get("maybe_description_rendered_html")?, - maybe_creator_user_token: row.try_get("maybe_creator_user_token")?, - maybe_creator_username: row.try_get("maybe_creator_username")?, - maybe_creator_display_name: row.try_get("maybe_creator_display_name")?, - maybe_creator_email_gravatar_hash: row.try_get("maybe_creator_email_gravatar_hash")?, - creator_ip_address: row.try_get("creator_ip_address")?, - creator_set_visibility: Visibility::try_from_mysql_row(row, "creator_set_visibility")?, - maybe_last_update_user_token: row.try_get("maybe_last_update_user_token")?, - original_download_url: row.try_get("original_download_url")?, - original_filename: row.try_get("original_filename")?, - file_size_bytes: row.try_get("file_size_bytes")?, - file_checksum_sha2: row.try_get("file_checksum_sha2")?, - public_bucket_hash: row.try_get("public_bucket_hash")?, - maybe_public_bucket_prefix: row.try_get("maybe_public_bucket_prefix")?, - maybe_public_bucket_extension: row.try_get("maybe_public_bucket_extension")?, - cached_usage_count: row.try_get("cached_usage_count")?, - maybe_cover_image_public_bucket_hash: row.try_get("maybe_cover_image_public_bucket_hash")?, - maybe_cover_image_public_bucket_prefix: row.try_get("maybe_cover_image_public_bucket_prefix")?, - maybe_cover_image_public_bucket_extension: row.try_get("maybe_cover_image_public_bucket_extension")?, - maybe_ratings_positive_count: row.try_get("maybe_ratings_positive_count")?, - maybe_ratings_negative_count: row.try_get("maybe_ratings_negative_count")?, - maybe_bookmark_count: row.try_get("maybe_bookmark_count")?, - version: row.try_get("version")?, - created_at: row.try_get("created_at")?, - updated_at: row.try_get("updated_at")?, - user_deleted_at: row.try_get("user_deleted_at")?, - mod_deleted_at: row.try_get("mod_deleted_at")?, - }) - } -} \ No newline at end of file diff --git a/crates/schema/database/mysql_queries/src/queries/model_weights/list/mod.rs b/crates/schema/database/mysql_queries/src/queries/model_weights/list/mod.rs index f4b83832be..faac84efeb 100644 --- a/crates/schema/database/mysql_queries/src/queries/model_weights/list/mod.rs +++ b/crates/schema/database/mysql_queries/src/queries/model_weights/list/mod.rs @@ -1,6 +1,5 @@ pub mod list_all_model_weight_tokens_for_backfill; pub mod list_all_model_weight_tokens_for_cached_usage_backfill; -pub mod list_featured_weights; pub mod list_model_weight_tokens_updated_since; pub mod list_model_weights_for_elastic_search_backfill_using_cursor; pub mod list_model_weights_for_text_to_speech; diff --git a/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/get_tts_inference_job_status.rs b/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/get_tts_inference_job_status.rs deleted file mode 100644 index 204d9dcbea..0000000000 --- a/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/get_tts_inference_job_status.rs +++ /dev/null @@ -1,74 +0,0 @@ -use anyhow::anyhow; -use chrono::{DateTime, Utc}; -use sqlx::MySqlPool; - -use errors::AnyhowResult; - -#[derive(Serialize)] -pub struct TtsInferenceJobStatusRecord { - pub job_token: String, - - pub status: String, - pub attempt_count: i32, - - pub maybe_result_token: Option, - pub maybe_public_bucket_wav_audio_path: Option, - - pub model_token: String, - pub tts_model_type: String, - pub title: String, // Name of the TTS model - - pub raw_inference_text: String, // User text - - pub created_at: DateTime, - pub updated_at: DateTime, -} - -pub async fn get_tts_inference_job_status( - tts_inference_job_token: &str, - mysql_pool: &MySqlPool -) -> AnyhowResult> { - - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_status = sqlx::query_as!( - TtsInferenceJobStatusRecord, - r#" -SELECT - jobs.token as job_token, - - jobs.status, - jobs.attempt_count, - jobs.on_success_result_token as maybe_result_token, - results.public_bucket_wav_audio_path as maybe_public_bucket_wav_audio_path, - - jobs.model_token, - tts.tts_model_type, - tts.title, - - jobs.raw_inference_text, - - jobs.created_at, - jobs.updated_at - -FROM tts_inference_jobs as jobs -JOIN tts_models as tts - ON tts.token = jobs.model_token -LEFT OUTER JOIN tts_results as results - ON jobs.on_success_result_token = results.token - -WHERE jobs.token = ? - "#, - tts_inference_job_token - ) - .fetch_one(mysql_pool) - .await; // TODO: This will return error if it doesn't exist - - match maybe_status { - Ok(record) => Ok(Some(record)), - Err(err) => match err { - sqlx::Error::RowNotFound => Ok(None), - _ => Err(anyhow!("tts job query error: {:?}", err)), - } - } -} diff --git a/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/mod.rs b/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/mod.rs index 6dcb6e3a78..6c5dc88909 100644 --- a/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/mod.rs +++ b/crates/schema/database/mysql_queries/src/queries/tts/tts_inference_jobs/mod.rs @@ -1,6 +1,5 @@ pub mod _keys; pub mod get_pending_tts_inference_job_detailed_stats; -pub mod get_tts_inference_job_status; pub mod insert_tts_inference_job; pub mod kill_tts_inference_jobs; pub mod list_available_tts_inference_jobs; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status_handler.rs index 2010564d67..96df7a6ed7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/get_tts_inference_job_status_handler.rs @@ -1,25 +1,15 @@ -use std::fmt; -use std::sync::Arc; - -use actix_web::error::ResponseError; use actix_web::web::Json; -use actix_web::http::StatusCode; use actix_web::web::Path; -use actix_web::{web, HttpMessage, HttpRequest}; use chrono::{DateTime, Utc}; -use log::error; -use redis::{Commands, RedisResult}; -use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status; -use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status; -use redis_common::redis_keys::RedisKeys; -use tokens::tokens::generic_inference_jobs::InferenceJobToken; - -use crate::http_server::web_utils::filter_model_name::filter_model_name; use crate::http_server::common_responses::common_web_error::CommonWebError; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::state::server_state::ServerState; + +/// Terminal status reported for every job so legacy clients stop polling. +const SYNTHETIC_JOB_STATUS: &str = "dead"; + +const SYNTHETIC_MODEL_TOKEN: &str = "synthetic_model_token"; +const SYNTHETIC_MODEL_TITLE: &str = "Legacy TTS is retired"; +const SYNTHETIC_MODEL_TYPE: &str = "tacotron2"; /// For the URL PathInfo #[derive(Deserialize)] @@ -27,6 +17,12 @@ pub struct GetTtsInferenceStatusPathInfo { token: String, } +#[derive(Serialize)] +pub struct GetTtsInferenceStatusSuccessResponse { + pub success: bool, + pub state: TtsInferenceJobStatusForResponse, +} + #[derive(Serialize)] pub struct TtsInferenceJobStatusForResponse { pub job_token: String, @@ -53,17 +49,14 @@ pub struct TtsInferenceJobStatusForResponse { pub updated_at: DateTime, } -#[derive(Serialize)] -pub struct GetTtsInferenceStatusSuccessResponse { - pub success: bool, - pub state: TtsInferenceJobStatusForResponse, -} -// NB: Not using derive_more::Display since Clion doesn't understand it. +/// RETIRED endpoint that still receives heavy zombie traffic. +/// +/// Legacy TTS inference is shut down and the job tables are no longer served. +/// Every job is reported as terminally "dead" (without touching MySQL or +/// Redis) so old clients stop polling. pub async fn get_tts_inference_job_status_handler( - http_request: HttpRequest, path: Path, - server_state: web::Data>) -> Result, CommonWebError> -{ +) -> Result, CommonWebError> { let job_token = path.into_inner().token; if job_token.trim() == "None" { @@ -72,156 +65,23 @@ pub async fn get_tts_inference_job_status_handler( return Err(CommonWebError::NotFound); } - // NB(bt,2023-11-27): We're moving TT2 over to `inference-job` (from `tts-inference-job`), which - // uses a wholly different job table. The token prefix determines the type of job: - // Legacy TT2 jobs (tts_inference_jobs) start with "JTINF:", and generic jobs start with "jinf_". - let record_for_response = if job_token.starts_with("jinf_") { - modern_lookup(&job_token, &server_state).await? - } else { - legacy_lookup(&job_token, &server_state).await? - }; + let now = Utc::now(); Ok(Json(GetTtsInferenceStatusSuccessResponse { success: true, - state: record_for_response, + state: TtsInferenceJobStatusForResponse { + job_token, + status: SYNTHETIC_JOB_STATUS.to_string(), + maybe_extra_status_description: None, + attempt_count: 1, + maybe_result_token: None, + maybe_public_bucket_wav_audio_path: None, + model_token: SYNTHETIC_MODEL_TOKEN.to_string(), + tts_model_type: SYNTHETIC_MODEL_TYPE.to_string(), + title: SYNTHETIC_MODEL_TITLE.to_string(), + raw_inference_text: String::new(), + created_at: now, + updated_at: now, + }, })) } - -async fn legacy_lookup(job_token: &str, server_state: &ServerState) - -> Result -{ - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_status = - get_tts_inference_job_status(&job_token, &server_state.mysql_pool).await; - - let record = match maybe_status { - Ok(Some(record)) => record, - Ok(None) => return Err(CommonWebError::NotFound), - Err(err) => { - error!("tts job query error: {:?}", err); - return Err(CommonWebError::from_anyhow_error(err)); - } - }; - - let mut redis = server_state.redis_pool - .get() - .map_err(|e| { - error!("redis error: {:?}", e); - CommonWebError::from_error(e) - })?; - - // TODO(bt,2023-05-21): Make async. - let extra_status_key = RedisKeys::tts_inference_extra_status_info(&job_token); - let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { - Ok(Some(status)) => { - Some(status) - }, - Ok(None) => None, - Err(e) => { - error!("redis error: {:?}", e); - None // Fail open - }, - }; - - Ok(TtsInferenceJobStatusForResponse { - job_token: record.job_token, - status: record.status, - maybe_extra_status_description, - attempt_count: record.attempt_count as u8, - maybe_result_token: record.maybe_result_token, - maybe_public_bucket_wav_audio_path: record.maybe_public_bucket_wav_audio_path, - model_token: record.model_token, - tts_model_type: record.tts_model_type, - title: record.title, - raw_inference_text: record.raw_inference_text, - created_at: record.created_at, - updated_at: record.updated_at, - }) -} - -async fn modern_lookup(job_token: &str, server_state: &ServerState) - -> Result -{ - // NB: Lookup failure is Err(RowNotFound). - // NB: Since this is publicly exposed, we don't query sensitive data. - let maybe_status = - get_tts_inference_job_status(&job_token, &server_state.mysql_pool).await; - - let job_token = InferenceJobToken::new_from_str(job_token); - - let maybe_status = get_inference_job_status(&job_token, &server_state.mysql_pool).await; - - let record = match maybe_status { - Ok(Some(record)) => record, - Ok(None) => return Err(CommonWebError::NotFound), - Err(err) => { - error!("tts job query error: {:?}", err); - return Err(CommonWebError::from_anyhow_error(err)); - } - }; - - let mut redis = server_state.redis_pool - .get() - .map_err(|e| { - error!("redis error: {:?}", e); - CommonWebError::from_error(e) - })?; - - // TODO(bt,2023-05-21): Make async. - let extra_status_key = RedisKeys::generic_inference_extra_status_info(job_token.as_str()); - let maybe_extra_status_value : RedisResult> = redis.get(&extra_status_key); - - let maybe_extra_status_description = match maybe_extra_status_value { - Err(e) => { - error!("redis error: {:?}", e); - None // Fail open - }, - Ok(maybe_value) => match maybe_value.as_deref() { - Some("1") => { - // TODO(bt,2023-10-20): Redis is reporting "1" and it's been surfacing this as a weird - // message to the frontend for months. This needs proper fixing. - None - }, - Some(value) => Some(value.to_string()), - None => None, - } - }; - - // NB: Model type is probably TT2, but let's filter it in case a hidden model type ever sneaks in - let model_type = record.request_details.maybe_model_type.as_deref().unwrap_or_else(|| "tacotron2"); - let model_type = filter_model_name(model_type); - - Ok(TtsInferenceJobStatusForResponse { - job_token: record.job_token.to_string(), - status: record.status.to_string(), - maybe_extra_status_description, - attempt_count: record.attempt_count as u8, - maybe_result_token: record.maybe_result_details.as_ref().map(|result| result.entity_token.clone()), - maybe_public_bucket_wav_audio_path: record.maybe_result_details.map(|result_details| { - match result_details.entity_type.as_str() { - "media_file" => { - // NB: We're migrating TTS to media_files. - // Zero shot TTS uses media files. - // Legacy TT2 uses old pathing. - MediaFileBucketPath::from_object_hash( - &result_details.public_bucket_location_or_hash, - result_details.maybe_media_file_public_bucket_prefix.as_deref(), - result_details.maybe_media_file_public_bucket_extension.as_deref()) - .get_full_object_path_str() - .to_string() - } - _ => { - // NB: TTS results receive the legacy treatment where their table only reports the full bucket path - result_details.public_bucket_location_or_hash - } - } - }), - model_token: record.request_details.maybe_model_token.unwrap_or_else(|| "NO_MODEL_TOKEN".to_string()), - tts_model_type: model_type, - title: record.request_details.maybe_model_title.unwrap_or_else(|| "no model title".to_string()), - raw_inference_text: record.request_details.maybe_raw_inference_text.unwrap_or_else(|| "no inference text".to_string()), - created_at: record.created_at, - updated_at: record.updated_at, - }) -} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list/list_featured_weights_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list/list_featured_weights_handler.rs index 47b03afc0e..a0bfbeec12 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list/list_featured_weights_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/list/list_featured_weights_handler.rs @@ -1,30 +1,17 @@ -use std::sync::Arc; - use actix_web::error::ResponseError; use actix_web::web::Json; use actix_web::http::StatusCode; use actix_web::web::Query; -use actix_web::{web, HttpRequest}; use chrono::{DateTime, Utc}; -use log::warn; use utoipa::{IntoParams, ToSchema}; -use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::model_weights::weights_category::WeightsCategory; -use enums::common::view_as::ViewAs; use enums::by_table::model_weights::weights_types::WeightsType; -use mysql_queries::queries::model_weights::list::list_featured_weights::{list_featured_weights, ListFeaturedWeightsArgs}; -use primitives::numerics::u64_to_u32_saturating::u64_to_u32_saturating; use tokens::tokens::model_weights::ModelWeightToken; use crate::http_server::common_responses::media::weights_cover_image_details::WeightsCoverImageDetails; use crate::http_server::common_responses::simple_entity_stats::SimpleEntityStats; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; -use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; -use crate::http_server::endpoints::weights::helpers::get_scoped_weights_categories::get_scoped_weights_categories; -use crate::http_server::endpoints::weights::helpers::get_scoped_weights_types::get_scoped_weights_types; -use crate::state::server_state::ServerState; -use crate::util::title_to_url_slug::title_to_url_slug; #[derive(Deserialize, ToSchema, IntoParams)] pub struct ListFeaturedWeightsQueryParams { @@ -101,9 +88,6 @@ pub struct FeaturedModelWeightForList { pub updated_at: DateTime, } -/// The key we store featured weights tokens under -const REDIS_KEY : &str = "featured_weights_list"; - #[derive(Debug, ToSchema)] pub enum ListFeaturedWeightsError { NotAuthorized, @@ -125,7 +109,10 @@ impl ResponseError for ListFeaturedWeightsError { } } -/// List model weights that the staff has selected as "the best" models on the site. +/// RETIRED: featured model weights are shut down. +/// +/// Always returns an empty result list (no database access) for the legacy +/// clients that still call this endpoint. #[utoipa::path( get, tag = "Model Weights", @@ -138,148 +125,10 @@ impl ResponseError for ListFeaturedWeightsError { ), )] pub async fn list_featured_weights_handler( - http_request: HttpRequest, - query: Query, - server_state: web::Data> -) -> Result, impl ResponseError> { - - let maybe_user_session = server_state - .session_checker - .maybe_get_user_session(&http_request, &server_state.mysql_pool) - .await - .map_err(|e| { - warn!("Session checker error: {:?}", e); - ListFeaturedWeightsError::ServerError - })?; - - let mut is_mod = false; - - let maybe_scoped_weights_types = get_scoped_weights_types(query.filter_weights_types.as_deref()); - let maybe_scoped_weights_categories = get_scoped_weights_categories(query.filter_weights_categories.as_deref()); - - match maybe_user_session { - None => {}, - Some(session) => { - is_mod = session.can_ban_users; - }, - }; - - // TODO(bt,2023-12-04): Enforce real maximums and defaults - let limit = query.page_size.unwrap_or(25); - - let sort_ascending = query.sort_ascending.unwrap_or(false); - let cursor_is_reversed = query.cursor_is_reversed.unwrap_or(false); - - let cursor = if let Some(cursor) = query.cursor.as_deref() { - let cursor = server_state.sort_key_crypto.decrypt_id(cursor) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListFeaturedWeightsError::ServerError - })?; - Some(cursor as usize) - } else { - None - }; - - let view_as = if is_mod { - ViewAs::Moderator - } else { - ViewAs::AnotherUser - }; - - let query_results = list_featured_weights(ListFeaturedWeightsArgs { - limit, - maybe_offset: cursor, - sort_ascending, - cursor_is_reversed, - view_as, - maybe_scoped_weight_types: maybe_scoped_weights_types.as_ref(), - maybe_scoped_weight_categories: maybe_scoped_weights_categories.as_ref(), - mysql_pool: &server_state.mysql_pool, - }).await; - - let results_page = match query_results { - Ok(results) => results, - Err(e) => { - warn!("Query error: {:?}", e); - return Err(ListFeaturedWeightsError::ServerError); - } - }; - - let cursor_next = if let Some(id) = results_page.last_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListFeaturedWeightsError::ServerError - })?; - Some(cursor) - } else { - None - }; - - let cursor_previous = if let Some(id) = results_page.first_id { - let cursor = server_state.sort_key_crypto.encrypt_id(id as u64) - .map_err(|e| { - warn!("crypto error: {:?}", e); - ListFeaturedWeightsError::ServerError - })?; - Some(cursor) - } else { - None - }; - - let media_domain = get_media_domain(&http_request); - - let response = ListFeaturedWeightsSuccessResponse { + _query: Query, +) -> Result, ListFeaturedWeightsError> { + Ok(Json(ListFeaturedWeightsSuccessResponse { success: true, - results: results_page.records.into_iter() - .map(|w| { - let cover_image_details = WeightsCoverImageDetails::from_optional_db_fields( - media_domain, - server_state.server_environment, - &w.token, - w.maybe_cover_image_public_bucket_hash.as_deref(), - w.maybe_cover_image_public_bucket_prefix.as_deref(), - w.maybe_cover_image_public_bucket_extension.as_deref(), - ); - - let maybe_cover_image = w.maybe_cover_image_public_bucket_hash - .as_deref() - .map(|hash| { - MediaFileBucketPath::from_object_hash( - hash, - w.maybe_cover_image_public_bucket_prefix.as_deref(), - w.maybe_cover_image_public_bucket_extension.as_deref()) - .get_full_object_path_str() - .to_string() - }); - - FeaturedModelWeightForList { - weight_token: w.token, - maybe_url_slug: title_to_url_slug(&w.title), - title: w.title, - maybe_ietf_language_tag: w.maybe_ietf_language_tag, - maybe_ietf_primary_language_subtag: w.maybe_ietf_primary_language_subtag, - weight_type: w.weights_type, - weight_category: w.weights_category, - cover_image: cover_image_details, - maybe_cover_image_public_bucket_path: maybe_cover_image, - creator: UserDetailsLight::from_optional_db_fields( - w.maybe_creator_user_token.as_ref(), - w.maybe_creator_username.as_deref(), - w.maybe_creator_display_name.as_deref(), - w.maybe_creator_email_gravatar_hash.as_deref(), - ), - stats: SimpleEntityStats { - positive_rating_count: w.maybe_ratings_positive_count.unwrap_or(0), - bookmark_count: w.maybe_bookmark_count.unwrap_or(0), - }, - usage_count: u64_to_u32_saturating(w.cached_usage_count), - created_at: w.created_at, - updated_at: w.updated_at, - } - }).collect::>(), - }; - - Ok(Json(response)) + results: Vec::new(), + })) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/search/search_model_weights_impl.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/search/search_model_weights_impl.rs index a0359c8834..1dee2a9db7 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/weights/search/search_model_weights_impl.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/weights/search/search_model_weights_impl.rs @@ -3,7 +3,6 @@ #![forbid(unused_mut)] #![forbid(unused_variables)] -use std::collections::HashSet; use std::fmt; use std::sync::Arc; @@ -12,23 +11,18 @@ use actix_web::http::StatusCode; use actix_web::web::Json; use actix_web::{web, HttpRequest, HttpResponse}; use chrono::{DateTime, Utc}; -use log::error; use utoipa::ToSchema; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; +use enums::common::visibility::Visibility; +use tokens::tokens::model_weights::ModelWeightToken; + use crate::http_server::common_responses::media::weights_cover_image_details::WeightsCoverImageDetails; use crate::http_server::common_responses::simple_entity_stats::SimpleEntityStats; use crate::http_server::common_responses::user_details_lite::UserDetailsLight; -use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::state::server_state::ServerState; -use crate::util::title_to_url_slug::title_to_url_slug; -use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use elasticsearch_schema::searches::search_model_weights::search_model_weights::{search_model_weights, ModelWeightsSortDirection, ModelWeightsSortField, SearchArgs}; -use enums::by_table::model_weights::weights_category::WeightsCategory; -use enums::by_table::model_weights::weights_types::WeightsType; -use enums::common::visibility::Visibility; -use primitives::numerics::i32_to_u32_zero_clamped::i32_to_u32_zero_clamped; -use tokens::tokens::model_weights::ModelWeightToken; #[derive(Deserialize, ToSchema)] pub struct SearchModelWeightsRequest { @@ -63,6 +57,12 @@ pub enum SearchModelWeightsSortDirection { Descending, } +#[derive(Serialize, ToSchema)] +pub struct SearchModelWeightsSuccessResponse { + pub success: bool, + pub weights: Vec, +} + #[derive(Serialize, Clone, ToSchema)] pub struct ModelWeightSearchResult { pub weight_token: ModelWeightToken, @@ -108,12 +108,6 @@ pub struct ModelWeightSearchResult { pub updated_at: DateTime, } -#[derive(Serialize, ToSchema)] -pub struct SearchModelWeightsSuccessResponse { - pub success: bool, - pub weights: Vec, -} - #[derive(Debug, ToSchema)] pub enum SearchModelWeightsError { ServerError, @@ -142,128 +136,18 @@ impl fmt::Display for SearchModelWeightsError { } } +/// RETIRED: model weight search is shut down. +/// +/// Always returns an empty result list (no Elasticsearch access) for the +/// legacy clients that still call the search endpoints. pub async fn search_model_weights_impl( - http_request: HttpRequest, - request: SearchModelWeightsRequest, - server_state: web::Data> + _http_request: HttpRequest, + _request: SearchModelWeightsRequest, + _server_state: web::Data> ) -> Result, SearchModelWeightsError> { - let maybe_weights_categories = request.weight_category - .map(|weight_category| { - let mut set = HashSet::new(); - set.insert(weight_category); - set - }); - - let maybe_weights_types = request.weight_type - .map(|weight_type| { - let mut set = HashSet::new(); - set.insert(weight_type); - set - }); - - let sort_field = match request.sort_field { - Some(SearchModelWeightsSortField::MatchScore) => Some(ModelWeightsSortField::MatchScore), - Some(SearchModelWeightsSortField::CreatedAt) => Some(ModelWeightsSortField::CreatedAt), - Some(SearchModelWeightsSortField::UsageCount) => Some(ModelWeightsSortField::UsageCount), - Some(SearchModelWeightsSortField::BookmarkCount) => Some(ModelWeightsSortField::BookmarkCount), - Some(SearchModelWeightsSortField::PositiveRatingCount) => Some(ModelWeightsSortField::PositiveRatingCount), - None => None, - }; - - let sort_direction = match request.sort_direction { - Some(SearchModelWeightsSortDirection::Ascending) => Some(ModelWeightsSortDirection::Ascending), - Some(SearchModelWeightsSortDirection::Descending) => Some(ModelWeightsSortDirection::Descending), - None => None, - }; - - let results = search_model_weights(SearchArgs { - search_term: &request.search_term, - maybe_creator_user_token: None, - maybe_ietf_primary_language_subtag: request.ietf_language_subtag.as_deref(), - maybe_weights_categories, - maybe_weights_types, - sort_field, - sort_direction, - minimum_score: request.minimum_score, - client: &server_state.elasticsearch, - }) - .await - .map_err(|err| { - error!("Searching error: {:?}", err); - SearchModelWeightsError::ServerError - })?; - - let media_domain = get_media_domain(&http_request); - - let results = results.into_iter() - .map(|result| { - let cover_image_details = WeightsCoverImageDetails::from_optional_db_fields( - media_domain, - server_state.server_environment, - &result.token, - result.maybe_cover_image_public_bucket_hash.as_deref(), - result.maybe_cover_image_public_bucket_prefix.as_deref(), - result.maybe_cover_image_public_bucket_extension.as_deref(), - ); - - let maybe_cover_image = result.maybe_cover_image_public_bucket_hash - .as_deref() - .map(|hash| { - MediaFileBucketPath::from_object_hash( - &hash, - result.maybe_cover_image_public_bucket_prefix.as_deref(), - result.maybe_cover_image_public_bucket_extension.as_deref()) - .get_full_object_path_str() - .to_string() - }); - - ModelWeightSearchResult { - weight_token: result.token, - weight_type: result.weights_type, - weight_category: result.weights_category, - maybe_url_slug: title_to_url_slug(&result.title), - title: result.title, - creator: UserDetailsLight::from_db_fields( - &result.creator_user_token, - &result.creator_username, - &result.creator_display_name, - &result.creator_gravatar_hash, - ), - cover_image: cover_image_details, - maybe_cover_image_public_bucket_path: maybe_cover_image, - is_featured: result.is_featured.unwrap_or(false), - stats: SimpleEntityStats { - positive_rating_count: result.ratings_positive_count, - bookmark_count: result.bookmark_count, - }, - usage_count: i32_to_u32_zero_clamped(result.cached_usage_count.unwrap_or(0)), - maybe_ietf_language_tag: result.maybe_ietf_language_tag, - maybe_ietf_primary_language_subtag: result.maybe_ietf_primary_language_subtag, - creator_set_visibility: result.creator_set_visibility, - created_at: result.created_at, - updated_at: result.updated_at, - } - }) - .collect::>(); - - // TODO(bt,2023-10-27): For some reason Elasticsearch returns duplicates. Maybe we populated the - // DB twice? Need to filter them out, or React chokes and gets stuck on duplicates. (Effectively - // freezing them into the UI, despite component updates) - - let mut added_tokens = HashSet::new(); - let mut new_results = Vec::with_capacity(results.len()); - - for result in results.into_iter() { - if added_tokens.contains(&result.weight_token) { - continue; - } - added_tokens.insert(result.weight_token.clone()); - new_results.push(result); - } - Ok(Json(SearchModelWeightsSuccessResponse { success: true, - weights: new_results, + weights: Vec::new(), })) }