synthetic categories

Cargo.lock

@@ -1529,6 +1529,14 @@ dependencies = [
  "toml",
 ]
 
+[[package]]
+name = "datetimes"
+version = "0.0.1"
+dependencies = [
+ "chrono",
+ "once_cell",
+]
+
 [[package]]
 name = "debug-helper"
 version = "0.3.11"
@@ -4597,11 +4605,13 @@ dependencies = [
  "container_common",
  "data-encoding",
  "database_queries",
+ "datetimes",
  "derive_more",
  "diesel",
  "dotenv",
  "easyenv",
  "enums",
+ "errors",
  "files",
  "futures 0.1.31",
  "futures 0.3.17",

Cargo.toml

@@ -23,6 +23,7 @@ members = [
   "crates/lib/container_common",
   "crates/lib/crockford",
   "crates/lib/database_queries",
+  "crates/lib/datetimes",
   "crates/lib/errors",
   "crates/lib/files",
   "crates/lib/google_drive_common",

crates/_types/enums/src/by_table/model_categories/model_type.rs

@@ -7,11 +7,11 @@ use strum::EnumCount;
 use strum::EnumIter;
 
 /// Used in the `model_categories` table in an `ENUM` field.
-/// (We want to stop using enum fields due to migration issues.)
+/// (*WE WANT TO STOP USING ENUM FIELDS DUE TO MIGRATION ISSUES*)
 ///
 /// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
 #[cfg_attr(test, derive(EnumIter, EnumCount))]
-#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize)]
+#[derive(Clone, Copy, Eq, PartialEq, Hash, Deserialize, Serialize)]
 #[serde(rename_all = "lowercase")]
 pub enum ModelType {
   Tts,

crates/lib/database_queries/src/queries/model_categories/list_categories_query_builder.rs

@@ -8,12 +8,12 @@ use sqlx::pool::PoolConnection;
 
 /// List of categories
 /// This is *NOT* paginated, as that defeats the purpose of having a category system.
-#[derive(Serialize)]
+#[derive(Serialize, Clone)]
 pub struct CategoryList {
   pub categories: Vec<Category>,
 }
 
-#[derive(Serialize)]
+#[derive(Serialize, Clone)]
 pub struct Category {
   pub category_token: String,
 

crates/lib/datetimes/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "datetimes"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "datetimes"
+path = "src/lib.rs"
+
+[dependencies]
+chrono = { version = "0.4.19", features = ["serde"] }
+once_cell = "1.9.0"
+
+[dev-dependencies]
+# (none)

crates/lib/datetimes/src/lib.rs

@@ -0,0 +1,40 @@
+//! datetimes
+//!
+//! While "chrono" is the preferred datetime crate, we still leverage some
+//! stuff in "time" and std. This crate contains some additional constants
+//! and utility methods around datetimes and durations not present in the
+//! primary 3rd party and standard libraries.
+//!
+
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+use chrono::{DateTime, NaiveDateTime, Utc};
+use once_cell::sync::Lazy;
+
+// NB: Chrono doesn't have any 'const fn's of not to leverage.
+/// This is the unix epoch datetime.
+pub static CHRONO_DATETIME_UNIX_EPOCH : Lazy<DateTime<Utc>> = Lazy::new(|| {
+  DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(0, 0), Utc)
+});
+
+#[cfg(test)]
+mod tests {
+  use crate::CHRONO_DATETIME_UNIX_EPOCH;
+
+  #[test]
+  fn test_chrono_datetime_unix_epoch() {
+    assert_eq!(CHRONO_DATETIME_UNIX_EPOCH.timestamp(), 0)
+  }
+}

crates/service/web/storyteller_web/Cargo.toml

@@ -21,7 +21,9 @@ cloud_storage = { path = "../../../lib/cloud_storage" }
 config = { path = "../../../lib/config" }
 container_common = { path = "../../../lib/container_common" }
 database_queries = { path = "../../../lib/database_queries" }
+datetimes = { path = "../../../lib/datetimes" }
 enums = { path = "../../../_types/enums" }
+errors = { path = "../../../lib/errors" }
 files = { path = "../../../lib/files" }
 google_drive_common = { path = "../../../lib/google_drive_common" }
 http_server_common = { path = "../../../lib/http_server_common" }

crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_fully_computed_assigned_tts_categories.rs

@@ -11,20 +11,24 @@ use actix_web::error::ResponseError;
 use actix_web::http::StatusCode;
 use actix_web::{web, HttpResponse, HttpRequest};
 use container_common::anyhow_result::AnyhowResult;
+use crate::model::cached_queries::list_cached_tts_categories_for_public_dropdown::list_cached_tts_categories_for_public_dropdown;
 use crate::server_state::ServerState;
-use database_queries::queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder;
+use database_queries::queries::model_categories::list_categories_query_builder::CategoryList;
 use database_queries::queries::tts::tts_category_assignments::fetch_and_build_tts_model_category_map::fetch_and_build_tts_model_category_map_with_connection;
 use database_queries::queries::tts::tts_models::list_tts_models::list_tts_models_with_connection;
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
 use lexical_sort::natural_lexical_cmp;
 use log::{info, error};
-use sqlx::{MySql, MySqlPool};
+use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
 use sqlx::pool::PoolConnection;
+use sqlx::{MySql, MySqlPool};
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::fmt;
 use std::sync::Arc;
+use chrono::{DateTime, Utc};
 use tokens::tokens::model_categories::ModelCategoryToken;
 use tokens::tokens::tts_models::TtsModelToken;
+use crate::model::categories::synthetic_category_list::SYNTHETIC_CATEGORY_LATEST_TTS_MODELS;
 
 // =============== Success Response ===============
 
@@ -109,7 +113,9 @@ pub async fn list_fully_computed_assigned_tts_categories_handler(
       category_assignments
     },
     None => {
-      let category_assignments = query_and_construct_payload(&server_state.mysql_pool)
+      let category_assignments = query_and_construct_payload(
+        &server_state.caches.database_tts_category_list,
+        &server_state.mysql_pool)
           .await?;
 
       server_state.caches.tts_model_category_assignments.store_copy(&category_assignments)
@@ -143,13 +149,16 @@ pub async fn list_fully_computed_assigned_tts_categories_handler(
 struct CategoryInfoLite {
   category_token: ModelCategoryToken,
   maybe_parent_category_token: Option<ModelCategoryToken>,
+  // Fields for sorting:
   category_name_for_sorting: String,
 }
 
 #[derive(Clone)]
 struct TtsModelInfoLite {
   model_token: TtsModelToken,
+  // Fields for sorting:
   title_for_sorting: String,
+  created_at: DateTime<Utc>,
 }
 
 #[derive(Clone)]
@@ -164,8 +173,11 @@ type ModelTokenToCategoryTokensMap = HashMap<TtsModelToken, HashSet<ModelCategor
 // NB: We use BTree to maintain insertion order for our return type.
 type CategoryTokenToModelTokensMap = BTreeMap<ModelCategoryToken, BTreeSet<TtsModelToken>>;
 
-async fn query_and_construct_payload(mysql_pool: &MySqlPool) -> Result<ModelTokensByCategoryToken, ListFullyComputedAssignedTtsCategoriesError> {
-  let (categories, model_category_map) = {
+async fn query_and_construct_payload(
+  category_cache: &SingleItemTtlCache<CategoryList>,
+  mysql_pool: &MySqlPool
+) -> Result<ModelTokensByCategoryToken, ListFullyComputedAssignedTtsCategoriesError> {
+  let (categories, models, model_category_map) = {
     let mut mysql_connection = mysql_pool.acquire()
         .await
         .map_err(|e| {
@@ -173,14 +185,14 @@ async fn query_and_construct_payload(mysql_pool: &MySqlPool) -> Result<ModelToke
           ListFullyComputedAssignedTtsCategoriesError::ServerError
         })?;
 
-    //let models = list_tts_models(&mut mysql_connection)
-    //    .await
-    //    .map_err(|e| {
-    //      error!("Error querying models: {:?}", e);
-    //      ListFullyComputedAssignedTtsCategoriesError::ServerError
-    //    })?;
+    let models = list_tts_models(&mut mysql_connection)
+        .await
+        .map_err(|e| {
+          error!("Error querying models: {:?}", e);
+          ListFullyComputedAssignedTtsCategoriesError::ServerError
+        })?;
 
-    let categories = list_tts_categories(&mut mysql_connection)
+    let categories = list_tts_categories(category_cache, &mut mysql_connection)
         .await
         .map_err(|e| {
           error!("Error querying categories: {:?}", e);
@@ -194,27 +206,28 @@ async fn query_and_construct_payload(mysql_pool: &MySqlPool) -> Result<ModelToke
           ListFullyComputedAssignedTtsCategoriesError::ServerError
         })?;
 
-    (categories, model_category_map)
+    (categories, models, model_category_map)
   };
 
+  let mut recursive_category_to_model_map = recursive_category_to_model_map(&model_category_map, &categories);
+
+  add_recent_models(&mut recursive_category_to_model_map, models);
+
   Ok(ModelTokensByCategoryToken {
-    recursive: recursive_category_to_model_map(&model_category_map, &categories),
+    recursive: recursive_category_to_model_map,
     //leaf_only: leaf_category_to_model_map(&model_category_map),
   })
 }
 
 // ========== Queries / model transformations ==========
 
-async fn list_tts_categories(mysql_connection: &mut PoolConnection<MySql>) -> AnyhowResult<Vec<CategoryInfoLite>> {
-  let query_builder = ListCategoriesQueryBuilder::new()
-      .show_deleted(false)
-      .show_unapproved(false)
-      .scope_model_type(Some("tts"));
+async fn list_tts_categories(
+  category_cache: &SingleItemTtlCache<CategoryList>,
+  mysql_connection: &mut PoolConnection<MySql>
+) -> AnyhowResult<Vec<CategoryInfoLite>> {
 
-  let categories = query_builder
-      .perform_query_using_connection(mysql_connection)
-      .await?
-      .categories;
+  let categories = list_cached_tts_categories_for_public_dropdown(category_cache, mysql_connection).await?;
+  let categories = categories.categories;
 
   let mut categories = categories.into_iter()
       .map(|c| CategoryInfoLite {
@@ -245,6 +258,7 @@ async fn list_tts_models(mysql_connection: &mut PoolConnection<MySql>) -> Anyhow
       .map(|m| TtsModelInfoLite {
         model_token: TtsModelToken::new_from_str(&m.model_token),
         title_for_sorting: m.title,
+        created_at: m.created_at,
       })
       .collect::<Vec<TtsModelInfoLite>>();
 
@@ -404,6 +418,23 @@ fn recursively_find_category_ancestors(
   }
 }
 
+fn add_recent_models(recursive_category_to_model_map: &mut CategoryTokenToModelTokensMap, models: Vec<TtsModelInfoLite>) {
+  let mut model_refs : Vec<&TtsModelInfoLite> = models.iter()
+      .collect::<Vec<&TtsModelInfoLite>>();
+
+  // Make the list nice for human readers.
+  model_refs.sort_by(|a, b| b.created_at.cmp(&a.created_at));
+
+  let mut model_tokens = BTreeSet::new();
+
+  model_refs.iter().take(30).for_each(|model| {
+    model_tokens.insert(model.model_token.clone());
+  });
+
+  let synthetic_token = ModelCategoryToken::new_from_str(SYNTHETIC_CATEGORY_LATEST_TTS_MODELS.category_token);
+  recursive_category_to_model_map.insert(synthetic_token, model_tokens);
+}
+
 #[cfg(test)]
 mod tests {
   // TODO: This really needs tests!

crates/service/web/storyteller_web/src/http_server/endpoints/categories/tts/list_tts_categories.rs

@@ -7,13 +7,16 @@ use actix_web::error::ResponseError;
 use actix_web::http::StatusCode;
 use actix_web::{web, HttpResponse, HttpRequest};
 use chrono::{DateTime, Utc};
+use crate::model::cached_queries::list_cached_tts_categories_for_public_dropdown::list_cached_tts_categories_for_public_dropdown_db_pool;
+use crate::model::categories::synthetic_category_list::SYNTHETIC_CATEGORY_LIST;
 use crate::server_state::ServerState;
-use database_queries::queries::model_categories::list_categories_query_builder::ListCategoriesQueryBuilder;
+use datetimes::CHRONO_DATETIME_UNIX_EPOCH;
+use enums::by_table::model_categories::model_type::ModelType;
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
-use lexical_sort::natural_lexical_cmp;
-use log::{info, warn, error};
+use log::error;
 use std::fmt;
 use std::sync::Arc;
+use tokens::tokens::model_categories::ModelCategoryToken;
 
 // =============== Success Response ===============
 
@@ -27,26 +30,28 @@ pub struct ListTtsCategoriesResponse {
 /// Used for the main TTS dropdown as well as the TTS edit/tagging UI
 #[derive(Serialize, Clone)]
 pub struct DisplayCategory {
-  pub category_token: String,
+  pub category_token: ModelCategoryToken,
+  pub maybe_super_category_token: Option<ModelCategoryToken>,
 
-  pub model_type: String, // TODO: ENUM
-
-  pub maybe_super_category_token: Option<String>,
-
-  pub can_directly_have_models: bool,
-  pub can_have_subcategories: bool,
-  pub can_only_mods_apply: bool,
+  pub model_type: ModelType,
 
   pub name: String,
 
   /// Instead of making the frontend deal with "maybe_dropdown_name", always use this:
   pub name_for_dropdown: String,
 
+  pub can_directly_have_models: bool,
+  pub can_have_subcategories: bool,
+  pub can_only_mods_apply: bool,
+
   // Moderator fields
   // It's okay to leak this since we do for assigned categories and for mods to see
   // which assigned categories might be invalid.
   pub is_mod_approved: Option<bool>,
 
+  /// False if the category lives in the database, true if the model was generated by code
+  pub is_synthetic: bool,
+
   pub created_at: DateTime<Utc>,
   pub updated_at: DateTime<Utc>,
   pub deleted_at: Option<DateTime<Utc>>,
@@ -86,76 +91,64 @@ pub async fn list_tts_categories_handler(
   _http_request: HttpRequest,
   server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, ListTtsCategoriesError>
 {
-  let maybe_categories = server_state.caches.category_list.copy_without_bump_if_unexpired()
+  let database_categories = list_cached_tts_categories_for_public_dropdown_db_pool(
+    &server_state.caches.database_tts_category_list, &server_state.mysql_pool)
+      .await
       .map_err(|e| {
-        error!("Error consulting cache: {:?}", e);
+        error!("error querying categories from db/cache: {:?}", e);
         ListTtsCategoriesError::ServerError
       })?;
 
-  let categories = match maybe_categories {
-    Some(categories) => {
-      info!("Serving TTS categories from cache");
-      categories
-    },
-    None => {
-      let mut mysql_connection = server_state.mysql_pool.acquire()
-          .await
-          .map_err(|e| {
-            warn!("Could not acquire DB pool: {:?}", e);
-            ListTtsCategoriesError::ServerError
-          })?;
-
-      let query_builder = ListCategoriesQueryBuilder::new()
-          .show_deleted(false)
-          .show_unapproved(false)
-          .scope_model_type(Some("tts"));
-
-      let query_result =
-          query_builder.perform_query_using_connection(&mut mysql_connection).await;
-
-      let results = match query_result {
-        Ok(results) => results,
-        Err(err) => {
-          warn!("DB error: {:?}", err);
-          return Err(ListTtsCategoriesError::ServerError);
+  // NB: These will be already sorted.
+  // Transform them to be display-appropriate.
+  let mut categories = database_categories.categories.iter()
+      .map(|c| {
+        DisplayCategory {
+          category_token: ModelCategoryToken::new_from_str(&c.category_token),
+          // NB: Unwrap default of TTS because we don't have other types yet.
+          model_type: ModelType::from_str(&c.model_type)
+              .unwrap_or(ModelType::Tts),
+          maybe_super_category_token: c.maybe_super_category_token
+              .as_deref()
+              .map(|t| ModelCategoryToken::new_from_str(t)),
+          can_directly_have_models: c.can_directly_have_models,
+          can_have_subcategories: c.can_have_subcategories,
+          can_only_mods_apply: c.can_only_mods_apply,
+          is_mod_approved: c.is_mod_approved,
+          is_synthetic: false,
+          name: c.name.clone(),
+          // NB: Always populate dropdown name instead of leaving it as an optional.
+          // This way we can always rely on it to be authoritative for dropdowns.
+          name_for_dropdown: c.maybe_dropdown_name
+              .clone()
+              .unwrap_or(c.name.clone()),
+          created_at: c.created_at.clone(),
+          updated_at: c.updated_at.clone(),
+          deleted_at: c.deleted_at.clone(),
         }
-      };
+      })
+      .collect::<Vec<DisplayCategory>>();
 
-      warn!("Number of categories: {:?}", results.categories.len());
-
-      let mut categories = results.categories.iter()
-          .map(|c| {
-            DisplayCategory {
-              category_token: c.category_token.clone(),
-              model_type: c.model_type.clone(),
-              maybe_super_category_token: c.maybe_super_category_token.clone(),
-              can_directly_have_models: c.can_directly_have_models,
-              can_have_subcategories: c.can_have_subcategories,
-              can_only_mods_apply: c.can_only_mods_apply,
-              is_mod_approved: c.is_mod_approved,
-              name: c.name.clone(),
-              name_for_dropdown: c.maybe_dropdown_name.clone().unwrap_or(c.name.clone()),
-              created_at: c.created_at.clone(),
-              updated_at: c.updated_at.clone(),
-              deleted_at: c.deleted_at.clone(),
-            }
-          })
-          .collect::<Vec<DisplayCategory>>();
-
-      // NB: This might produce weird sorting resorts relative to the "name" field,
-      // but the typical way this should be consumed is via dropdowns.
-      categories.sort_by(|c1, c2|
-          natural_lexical_cmp(&c1.name_for_dropdown, &c2.name_for_dropdown));
-
-      server_state.caches.category_list.store_copy(&categories)
-          .map_err(|e| {
-            error!("Error storing cache: {:?}", e);
-            ListTtsCategoriesError::ServerError
-          })?;
-
-      categories
-    },
-  };
+  // Add the synthetic categories at bottom.
+  SYNTHETIC_CATEGORY_LIST.iter()
+      .for_each(|category| {
+        categories.push(DisplayCategory {
+          category_token: ModelCategoryToken::new_from_str(category.category_token),
+          maybe_super_category_token: category.maybe_super_category_token
+              .map(|token| ModelCategoryToken::new_from_str(token)),
+          model_type: category.model_type.clone(),
+          name: category.name.to_string(),
+          name_for_dropdown: category.name_for_dropdown.to_string(),
+          can_directly_have_models: category.can_directly_have_models,
+          can_have_subcategories: category.can_directly_have_subcategories,
+          can_only_mods_apply: true,
+          is_mod_approved: None,
+          is_synthetic: true,
+          created_at: CHRONO_DATETIME_UNIX_EPOCH.clone(),
+          updated_at: CHRONO_DATETIME_UNIX_EPOCH.clone(),
+          deleted_at: None,
+        });
+      });
 
   let response = ListTtsCategoriesResponse {
     success: true,

crates/service/web/storyteller_web/src/main.rs

@@ -259,7 +259,10 @@ async fn main() -> AnyhowResult<()> {
   let voice_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("VOICE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60));
   let category_list_cache_ttl = easyenv::get_env_duration_seconds_or_default("CATEGORY_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60));
   let voice_list_cache = SingleItemTtlCache::create_with_duration(voice_list_cache_ttl);
-  let category_list_cache = SingleItemTtlCache::create_with_duration(category_list_cache_ttl);
+
+  let database_tts_category_list_cache = SingleItemTtlCache::create_with_duration(
+    easyenv::get_env_duration_seconds_or_default("DATABASE_TTS_CATEGORY_LIST_CACHE_TTL_SECONDS", Duration::from_secs(60))
+  );
 
   let w2l_template_cache = SingleItemTtlCache::create_with_duration(
     easyenv::get_env_duration_seconds_or_default("W2L_TEMPLATE_LIST_CACHE_TTL_SECONDS", Duration::from_secs(300)) // 5 minutes
@@ -387,7 +390,7 @@ async fn main() -> AnyhowResult<()> {
     caches: InMemoryCaches {
       voice_list: voice_list_cache,
       w2l_template_list: w2l_template_cache,
-      category_list: category_list_cache,
+      database_tts_category_list: database_tts_category_list_cache,
       tts_queue_length: tts_queue_length_cache,
       tts_model_category_assignments: tts_model_category_assignments_cache,
     },

crates/service/web/storyteller_web/src/model/cached_queries/list_cached_tts_categories_for_public_dropdown.rs

@@ -0,0 +1,63 @@
+use anyhow::anyhow;
+use database_queries::queries::model_categories::list_categories_query_builder::{CategoryList, ListCategoriesQueryBuilder};
+use errors::AnyhowResult;
+use lexical_sort::natural_lexical_cmp;
+use log::{error};
+use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
+use sqlx::{MySql, MySqlPool};
+use sqlx::pool::PoolConnection;
+
+/// Fetch from cache if available, otherwise fetch from DB.
+pub async fn list_cached_tts_categories_for_public_dropdown_db_pool(
+  cache: &SingleItemTtlCache<CategoryList>,
+  mysql_pool: &MySqlPool,
+) -> AnyhowResult<CategoryList> {
+  let mut mysql_connection = mysql_pool.acquire().await?;
+  list_cached_tts_categories_for_public_dropdown(cache, &mut mysql_connection).await
+}
+
+/// Fetch from cache if available, otherwise fetch from DB.
+pub async fn list_cached_tts_categories_for_public_dropdown(
+  cache: &SingleItemTtlCache<CategoryList>,
+  mysql_connection: &mut PoolConnection<MySql>,
+) -> AnyhowResult<CategoryList> {
+  let maybe_database_categories = cache.copy_without_bump_if_unexpired()?;
+
+  match maybe_database_categories {
+    Some(categories) => Ok(categories),
+    None => {
+      // NB: Scope to just public TTS categories used in the dropdown
+      let query_builder = ListCategoriesQueryBuilder::new()
+          .show_deleted(false)
+          .show_unapproved(false)
+          .scope_model_type(Some("tts"));
+
+      let query_result =
+          query_builder.perform_query_using_connection(mysql_connection).await;
+
+      let mut results : CategoryList = match query_result {
+        Ok(results) => results,
+        Err(err) => {
+          error!("DB error: {:?}", err);
+          return Err(anyhow!("database error"));
+        }
+      };
+
+      // NB: This might produce weird sorting resorts relative to the "name" field,
+      // but the typical way this should be consumed is via dropdowns.
+      results.categories.sort_by(|c1, c2|
+          natural_lexical_cmp(
+            &c1.maybe_dropdown_name.as_deref().unwrap_or(&c1.name),
+            &c2.maybe_dropdown_name.as_deref().unwrap_or(&c2.name),
+          ));
+
+      cache.store_copy(&results)
+          .map_err(|e| {
+            error!("Error storing cache: {:?}", e);
+            anyhow!("cache save error")
+          })?;
+
+      Ok(results)
+    },
+  }
+}

crates/service/web/storyteller_web/src/model/cached_queries/mod.rs

@@ -0,0 +1 @@
+pub mod list_cached_tts_categories_for_public_dropdown;

crates/service/web/storyteller_web/src/model/categories/synthetic_category.rs

@@ -1,3 +1,4 @@
+use enums::by_table::model_categories::model_type::ModelType;
 use tokens::tokens::model_categories::ModelCategoryToken;
 
 #[derive(Clone, Eq, PartialEq, Hash, Debug)]
@@ -9,7 +10,7 @@ pub struct SyntheticCategory {
   pub name: &'static str,
   pub name_for_dropdown: &'static str,
 
-  pub model_type: &'static str,
+  pub model_type: ModelType,
 
   pub can_directly_have_models: bool,
   pub can_directly_have_subcategories: bool,

crates/service/web/storyteller_web/src/model/categories/synthetic_category_list.rs

@@ -1,22 +1,36 @@
-use tokens::tokens::model_categories::ModelCategoryToken;
+use chrono::{DateTime, Utc};
 use crate::model::categories::synthetic_category::SyntheticCategory;
+use enums::by_table::model_categories::model_type::ModelType;
+use once_cell::sync::Lazy;
+use tokens::tokens::model_categories::ModelCategoryToken;
 
-const CATEGORY_LATEST_TTS_MODELS : SyntheticCategory = SyntheticCategory {
-  category_token: "CAT:SYNTHETIC_LATEST_MODELS",
+pub const SYNTHETIC_CATEGORY_LATEST_TTS_MODELS : SyntheticCategory = SyntheticCategory {
+  category_token: "SYNTHETIC_CATEGORY:LATEST_MODELS",
   maybe_super_category_token: None,
-  name: "",
-  name_for_dropdown: "",
-  model_type: "tts",
+  name: "Latest Models",
+  name_for_dropdown: "Latest Models",
+  model_type: ModelType::Tts,
   can_directly_have_models: false,
   can_directly_have_subcategories: false,
 };
 
-const CATEGORY_TRENDING_TTS_MODELS : SyntheticCategory = SyntheticCategory {
-  category_token: "CAT:SYNTHETIC_TRENDING_MODELS",
+pub const SYNTHETIC_CATEGORY_TRENDING_TTS_MODELS : SyntheticCategory = SyntheticCategory {
+  category_token: "SYNTHETIC_CATEGORY:TRENDING_MODELS",
   maybe_super_category_token: None,
-  name: "",
-  name_for_dropdown: "",
-  model_type: "",
+  name: "Trending Models",
+  name_for_dropdown: "Trending Models",
+  model_type: ModelType::Tts,
   can_directly_have_models: false,
   can_directly_have_subcategories: false,
 };
+
+/// List of the system's statically configured synthetic TTS categories
+/// These are populated serverside and served to the frontend along with
+/// the user-generated/user-populated categories.
+pub static SYNTHETIC_CATEGORY_LIST : Lazy<Vec<&'static SyntheticCategory>> =
+  Lazy::new(|| {
+    Vec::from([
+      &SYNTHETIC_CATEGORY_LATEST_TTS_MODELS,
+      &SYNTHETIC_CATEGORY_TRENDING_TTS_MODELS,
+    ])
+  });

crates/service/web/storyteller_web/src/server_state.rs

@@ -16,6 +16,7 @@ use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
 use r2d2_redis::{r2d2, RedisConnectionManager};
 use reusable_types::server_environment::ServerEnvironment;
 use sqlx::MySqlPool;
+use database_queries::queries::model_categories::list_categories_query_builder::CategoryList;
 use url_config::third_party_url_redirector::ThirdPartyUrlRedirector;
 use users_component::utils::session_checker::SessionChecker;
 use users_component::utils::session_cookie_manager::SessionCookieManager;
@@ -118,26 +119,28 @@ pub struct RedisRateLimiters {
   pub model_upload: RedisRateLimiter,
 }
 
-/// In-memory caches
+/// In-memory caches with TTL-based eviction.
 #[derive(Clone)]
 pub struct InMemoryCaches {
-  /// In-memory caches with TTL-based eviction. Contains a list of all voices.
+  /// Contains a list of all voices.
   pub voice_list: SingleItemTtlCache<Vec<TtsModelRecordForResponse>>,
 
   /// Contains a list of all W2L templates.
   pub w2l_template_list: SingleItemTtlCache<Vec<W2lTemplateRecordForList>>,
 
-  /// In-memory caches with TTL-based eviction. Contains a list of all TTS categories.
-  pub category_list: SingleItemTtlCache<Vec<DisplayCategory>>,
+  /// Contains a list of all TTS categories in the database
+  /// (before any enrichment with synthetic categories)
+  /// This is used in several places (list categories, computed category assignments)
+  pub database_tts_category_list: SingleItemTtlCache<CategoryList>,
+
+  /// Computed category assignments for TTS models
+  /// This is approximately O(n^3) and recursively generates all super-category membership.
+  pub tts_model_category_assignments: SingleItemTtlCache<ModelTokensByCategoryToken>,
 
   /// TTS queue length
   /// The frontend will consult a distributed cache and use the monotonic DB time as a
   /// vector clock.
   pub tts_queue_length: SingleItemTtlCache<TtsQueueLengthResult>,
-
-  /// Computed category assignments for TTS models
-  /// This is approximately O(n^3) and recursively generates all super-category membership.
-  pub tts_model_category_assignments: SingleItemTtlCache<ModelTokensByCategoryToken>,
 }
 
 #[derive(Clone)]