From 2ea0e1450c8f606f9a5c30c5b07a03d050fc2cd4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 18 Jan 2023 00:40:56 -0500 Subject: [PATCH] Sort tts models returned by category membership endpoint --- .../add_synthetic_categories.rs | 10 ++-- .../error.rs | 1 - ..._fully_computed_assigned_tts_categories.rs | 41 +++------------ .../mod.rs | 3 +- .../query_and_construct_payload.rs | 40 ++++++++------- .../recursively_build_category_map.rs | 6 +-- .../sort_models.rs | 50 +++++++++++++++++++ ...ched_tts_categories_for_public_dropdown.rs | 2 +- 8 files changed, 87 insertions(+), 66 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/sort_models.rs diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/add_synthetic_categories.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/add_synthetic_categories.rs index f5bdee0e9a..bcfa0c8a78 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/add_synthetic_categories.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/add_synthetic_categories.rs @@ -3,17 +3,15 @@ #![forbid(unused_mut)] #![forbid(unused_variables)] -use std::collections::BTreeSet; - +use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::query_and_construct_payload::{CategoryTokenToModelTokensMap, TtsModelInfoLite}; +use crate::model::categories::synthetic_category_list::{SYNTHETIC_CATEGORY_LATEST_TTS_MODELS, SYNTHETIC_CATEGORY_TRENDING_TTS_MODELS}; use database_queries::queries::trending_model_analytics::list_trending_tts_models::TrendingModels; use enums::by_table::trending_model_analytics::window_name::WindowName; +use std::collections::BTreeSet; use tokens::tokens::model_categories::ModelCategoryToken; use tokens::tokens::tts_models::TtsModelToken; -use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::query_and_construct_payload::{CategoryTokenToModelTokensMap, TtsModelInfoLite}; -use crate::model::categories::synthetic_category_list::{SYNTHETIC_CATEGORY_LATEST_TTS_MODELS, SYNTHETIC_CATEGORY_TRENDING_TTS_MODELS}; - -pub fn add_recent_models(recursive_category_to_model_map: &mut CategoryTokenToModelTokensMap, models: Vec) { +pub fn add_recent_models(recursive_category_to_model_map: &mut CategoryTokenToModelTokensMap, models: &Vec) { let mut model_refs : Vec<&TtsModelInfoLite> = models.iter() .collect::>(); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/error.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/error.rs index 7a01d6df25..23809e9989 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/error.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/error.rs @@ -5,7 +5,6 @@ use actix_web::{HttpResponse, ResponseError}; use actix_web::http::StatusCode; - use http_server_common::response::serialize_as_json_error::serialize_as_json_error; #[derive(Debug, Serialize)] diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/list_fully_computed_assigned_tts_categories.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/list_fully_computed_assigned_tts_categories.rs index b92a6e0be0..00fd56cb67 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/list_fully_computed_assigned_tts_categories.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/list_fully_computed_assigned_tts_categories.rs @@ -7,18 +7,15 @@ //! belongs to. This saves an enormous amount of clientside CPU compute. //! -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; - use actix_web::{HttpRequest, HttpResponse, web}; -use log::{error, info}; - -use tokens::tokens::model_categories::ModelCategoryToken; -use tokens::tokens::tts_models::TtsModelToken; - use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::error::ListFullyComputedAssignedTtsCategoriesError; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::query_and_construct_payload::query_and_construct_payload; use crate::server_state::ServerState; +use log::{error, info}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokens::tokens::model_categories::ModelCategoryToken; +use tokens::tokens::tts_models::TtsModelToken; // =============== Success Response =============== @@ -26,37 +23,15 @@ use crate::server_state::ServerState; pub struct ListFullyComputedAssignedTtsCategoriesResponse { pub success: bool, -// /// All category tokens in use by at least one TTS model. -// /// Unused categories will not be present. -// pub utilized_tts_category_tokens: UtilizedCategoryTokens, - /// Maps of category tokens to the TTS model tokens that are assigned to them. pub category_token_to_tts_model_tokens: ModelTokensByCategoryToken, } -//#[derive(Serialize)] -//pub struct UtilizedCategoryTokens { -// /// Every category token used at least once by a TTS model (recursive). -// /// This is a recursive membership, so parent categories with no models assigned will still be -// /// present if at least one model is assigned to any of that category's children. -// pub recursive: BTreeSet, -// -// /// Every category token used at least once by a TTS model (leaf only). -// /// The TTS model must have a direct attachment to the category to be considered "assigned", -// /// so parent categories will not be considered "utilized" in this sense unless they have models -// /// directly assigned to them. -// pub leaf_only: BTreeSet, -//} - #[derive(Serialize, Clone)] pub struct ModelTokensByCategoryToken { /// For every category, the TTS model tokens assigned. This is built up recursively. /// Parent categories *will* include all of the TTS models assigned to children categories. - pub recursive: BTreeMap>, - -// /// For every category, the TTS model tokens *directly* assigned. -// /// Parent categories *will not* include the TTS models assigned to children categories. -// pub leaf_only: BTreeMap>, + pub recursive: BTreeMap>, } // =============== Handler =============== @@ -94,10 +69,6 @@ pub async fn list_fully_computed_assigned_tts_categories_handler( let response = ListFullyComputedAssignedTtsCategoriesResponse { success: true, - //utilized_tts_category_tokens: UtilizedCategoryTokens { - // recursive: recursive_category_tokens(&model_category_map, &categories), - // leaf_only: leaf_category_tokens(&model_category_map), - //}, category_token_to_tts_model_tokens: category_assignments, }; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/mod.rs index 4a342b2c8a..261f7a9d1b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/mod.rs @@ -2,4 +2,5 @@ pub mod add_synthetic_categories; pub mod error; pub mod list_fully_computed_assigned_tts_categories; pub mod query_and_construct_payload; -pub mod recursively_build_category_map; \ No newline at end of file +pub mod recursively_build_category_map; +pub mod sort_models; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/query_and_construct_payload.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/query_and_construct_payload.rs index afe9b5c1c2..2b8aa93c9e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/query_and_construct_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/query_and_construct_payload.rs @@ -3,28 +3,26 @@ #![forbid(unused_mut)] #![forbid(unused_variables)] -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - use chrono::{DateTime, Utc}; -use lexical_sort::natural_lexical_cmp; -use log::error; -use sqlx::{MySql, MySqlPool}; -use sqlx::pool::PoolConnection; - -use database_queries::queries::model_categories::list_categories_query_builder::CategoryList; -use database_queries::queries::trending_model_analytics::list_trending_tts_models::list_trending_tts_models; -use database_queries::queries::tts::tts_category_assignments::fetch_and_build_tts_model_category_map::fetch_and_build_tts_model_category_map_with_connection; -use database_queries::queries::tts::tts_models::list_tts_models::list_tts_models_with_connection; -use errors::AnyhowResult; -use memory_caching::single_item_ttl_cache::SingleItemTtlCache; -use tokens::tokens::model_categories::ModelCategoryToken; -use tokens::tokens::tts_models::TtsModelToken; - use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::add_synthetic_categories::{add_recent_models, add_trending_models}; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::error::ListFullyComputedAssignedTtsCategoriesError; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::list_fully_computed_assigned_tts_categories::ModelTokensByCategoryToken; use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::recursively_build_category_map::recursive_category_to_model_map; use crate::model::cached_queries::list_cached_tts_categories_for_public_dropdown::list_cached_tts_categories_for_public_dropdown; +use database_queries::queries::model_categories::list_categories_query_builder::CategoryList; +use database_queries::queries::trending_model_analytics::list_trending_tts_models::list_trending_tts_models; +use database_queries::queries::tts::tts_category_assignments::fetch_and_build_tts_model_category_map::fetch_and_build_tts_model_category_map_with_connection; +use database_queries::queries::tts::tts_models::list_tts_models::list_tts_models_with_connection; +use errors::AnyhowResult; +use lexical_sort::natural_lexical_cmp; +use log::error; +use memory_caching::single_item_ttl_cache::SingleItemTtlCache; +use sqlx::pool::PoolConnection; +use sqlx::{MySql, MySqlPool}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use tokens::tokens::model_categories::ModelCategoryToken; +use tokens::tokens::tts_models::TtsModelToken; +use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::sort_models::sort_models; #[derive(Clone)] pub struct CategoryInfoLite { @@ -107,11 +105,17 @@ pub async fn query_and_construct_payload( &categories); // Add synthetic categories - add_recent_models(&mut recursive_category_to_model_map, models); + add_recent_models(&mut recursive_category_to_model_map, &models); add_trending_models(&mut recursive_category_to_model_map, trending_models); + // TODO: This is a mess, because late in the process I realized "BTreeMap"/"BTreeSet" sorts on + // key and not insertion order. Classic reading the docs. + + // Sort all of the models within the categories + let final_map = sort_models(&recursive_category_to_model_map, &models); + Ok(ModelTokensByCategoryToken { - recursive: recursive_category_to_model_map, + recursive: final_map, }) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/recursively_build_category_map.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/recursively_build_category_map.rs index 6c7911b88b..26ec46c6da 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/recursively_build_category_map.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/recursively_build_category_map.rs @@ -3,11 +3,9 @@ #![forbid(unused_mut)] #![forbid(unused_variables)] -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - -use tokens::tokens::model_categories::ModelCategoryToken; - use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::query_and_construct_payload::{CategoryInfoLite, CategoryTokenToCategoryMap, CategoryTokenToModelTokensMap, ModelTokenToCategoryTokensMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use tokens::tokens::model_categories::ModelCategoryToken; fn recursive_category_tokens(model_category_map: &ModelTokenToCategoryTokensMap, all_categories: &Vec) -> BTreeSet { let mut category_tokens = leaf_category_tokens(model_category_map); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/sort_models.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/sort_models.rs new file mode 100644 index 0000000000..b6d0835010 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories/sort_models.rs @@ -0,0 +1,50 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use lexical_sort::natural_lexical_cmp; +use tokens::tokens::model_categories::ModelCategoryToken; +use tokens::tokens::tts_models::TtsModelToken; +use crate::http_server::endpoints::categories::tts::list_fully_computed_assigned_tts_categories::query_and_construct_payload::{CategoryTokenToModelTokensMap, TtsModelInfoLite}; + +const UNNAMED_MODEL_SORT_VALUE : &'static str = "unnamed"; + +struct SortableModelToken { + model_token: TtsModelToken, + model_sortable_name: String, +} + +pub fn sort_models( + category_to_model_map: &CategoryTokenToModelTokensMap, + models: &Vec +) -> BTreeMap> { + let models_by_token = models.into_iter() + .map(|model| (model.model_token.clone(), model)) + .collect::>(); + + let mut results = BTreeMap::new(); + + for (category_token, model_tokens) in category_to_model_map.iter() { + let mut sorted_model_tokens = model_tokens.iter() + .map(|model_token| { + let model_sortable_name = models_by_token.get(model_token) + .map(|model| model.title_for_sorting.to_string()) + .unwrap_or(UNNAMED_MODEL_SORT_VALUE.to_string()) + .to_lowercase(); + + SortableModelToken { + model_token: model_token.clone(), + model_sortable_name, + } + }) + .collect::>(); + + sorted_model_tokens.sort_by(|a, b| + natural_lexical_cmp(&a.model_sortable_name, &b.model_sortable_name)); + + let sorted_model_tokens = sorted_model_tokens.into_iter() + .map(|token_info| token_info.model_token) + .collect::>(); + + results.insert(category_token.clone(), sorted_model_tokens); + } + + results +} diff --git a/crates/service/web/storyteller_web/src/model/cached_queries/list_cached_tts_categories_for_public_dropdown.rs b/crates/service/web/storyteller_web/src/model/cached_queries/list_cached_tts_categories_for_public_dropdown.rs index 7795669052..00485fa2e9 100644 --- a/crates/service/web/storyteller_web/src/model/cached_queries/list_cached_tts_categories_for_public_dropdown.rs +++ b/crates/service/web/storyteller_web/src/model/cached_queries/list_cached_tts_categories_for_public_dropdown.rs @@ -4,8 +4,8 @@ use errors::AnyhowResult; use lexical_sort::natural_lexical_cmp; use log::{error}; use memory_caching::single_item_ttl_cache::SingleItemTtlCache; -use sqlx::{MySql, MySqlPool}; use sqlx::pool::PoolConnection; +use sqlx::{MySql, MySqlPool}; /// Fetch from cache if available, otherwise fetch from DB. pub async fn list_cached_tts_categories_for_public_dropdown_db_pool(