Use server precomputed category assignments

packages/components/src/api/ApiConfig.ts

@@ -326,6 +326,7 @@ class ApiConfig {
   }
 
   listTtsCategories(): string {
+    // TODO: Move to /v1
     return `${this.getScheme()}://${this.getNewApiHost()}/category/list/tts`;
   }
 
@@ -333,6 +334,10 @@ class ApiConfig {
     return `${this.getScheme()}://${this.getNewApiHost()}/category/assignments/tts/${ttsModelToken}`;
   }
 
+  getComputedTtsCategoryAssignments(): string {
+    return `${this.getScheme()}://${this.getNewApiHost()}/v1/category/computed_assignments/tts`;
+  }
+
   firehoseEvents(): string {
     return `${this.getScheme()}://${this.getNewApiHost()}/events`;
   }

packages/fakeyou.com/src/App.tsx

@@ -38,6 +38,7 @@ import {
   GetRetrievalJobStatus,
   GetRetrievalJobStatusIsOk,
 } from "@storyteller/components/src/api/retrieval/GetRetrievalJobStatus";
+import { GetComputedTtsCategoryAssignmentsSuccessResponse } from "./v2/api/category/GetComputedTtsCategoryAssignments";
 
 i18n
   .use(initReactI18next) // passes i18n down to react-i18next
@@ -73,6 +74,9 @@ interface Props {
   allTtsModels: TtsModelListItem[];
   setAllTtsModels: (allTtsModels: TtsModelListItem[]) => void;
 
+  computedTtsCategoryAssignments?: GetComputedTtsCategoryAssignmentsSuccessResponse;
+  setComputedTtsCategoryAssignments: (categoryAssignments: GetComputedTtsCategoryAssignmentsSuccessResponse) => void;
+
   allTtsCategoriesByTokenMap: Map<string, TtsCategoryType>;
   allTtsModelsByTokenMap: Map<string, TtsModelListItem>;
   ttsModelsByCategoryToken: Map<string, Set<TtsModelListItem>>;
@@ -704,6 +708,8 @@ class App extends React.Component<Props, State> {
                     setTtsModels={this.props.setAllTtsModels}
                     allTtsCategories={this.props.allTtsCategories}
                     setAllTtsCategories={this.props.setAllTtsCategories}
+                    computedTtsCategoryAssignments={this.props.computedTtsCategoryAssignments}
+                    setComputedTtsCategoryAssignments={this.props.setComputedTtsCategoryAssignments}
                     allTtsCategoriesByTokenMap={
                       this.props.allTtsCategoriesByTokenMap
                     }

packages/fakeyou.com/src/AppWrapper.tsx

@@ -3,6 +3,7 @@ import { TtsCategory } from './v2/api/category/ListTtsCategories';
 import { TtsModelListItem } from '@storyteller/components/src/api/tts/ListTtsModels';
 import { App } from './App';
 import { v4 as uuidv4 } from 'uuid';
+import { GetComputedTtsCategoryAssignmentsSuccessResponse } from './v2/api/category/GetComputedTtsCategoryAssignments';
 
 interface Props {
   // Certan browsers (iPhone) have pitiful support for drawing APIs. Worse yet,
@@ -41,6 +42,7 @@ export function AppWrapper(props: Props) {
   // These may be triggered by a different page than the user initially lands on.
   const [allTtsCategories, setAllTtsCategories] = useState<TtsCategoryType[]>([]);
   const [allTtsModels, setAllTtsModels] = useState<TtsModelListItem[]>([]);
+  const [computedTtsCategoryAssignments, setComputedTtsCategoryAssignments] = useState<GetComputedTtsCategoryAssignmentsSuccessResponse|undefined>(undefined);
 
   // Precalculated maps for lookup by primary key
   const [allCategoriesByTokenMap, setAllCategoriesByTokenMap] = useState<Map<string,TtsCategoryType>>(new Map());
@@ -90,37 +92,61 @@ export function AppWrapper(props: Props) {
     const rootLevel = [rootCategories];
     setDropdownCategories(rootLevel);
 
-    // Voice lookup table
-    let categoriesToTtsModelTokens = new Map();
-    // Category ancestry memoization
-    let categoryTokenToAllAncestorTokens : Map<string, Set<string>> = new Map();
+///  NB: This was the really expensive cubic time++ aglorithm to recursively calculate nested category assignments.
+///   This has since been ported to the server and will no longer be computed here. We're preserving this for posterity.
+///     
+///    // Voice lookup table
+///    let categoriesToTtsModelTokens = new Map();
+///    // Category ancestry memoization
+///    let categoryTokenToAllAncestorTokens : Map<string, Set<string>> = new Map();
+///
+///    // N * M with memoization should't be too bad here.
+///    // Also note that the models should be lexographically sorted by name.
+///    allTtsModels.forEach(ttsModel => {
+///      if (ttsModel.category_tokens.length === 0) {
+///        // TODO: Attach to "uncategorized" special category
+///        return;
+///      }
+///      ttsModel.category_tokens.forEach(categoryToken => {
+///        let ancestors = categoryTokenToAllAncestorTokens.get(categoryToken);
+///        if (ancestors === undefined) {
+///          ancestors = findAllAncestorTokens(categoryToken, categoriesByTokenMap);
+///          categoryTokenToAllAncestorTokens.set(categoryToken, ancestors);
+///        }
+///        ancestors.forEach(categoryToken => {
+///          let models : Set<TtsModelListItem> = categoriesToTtsModelTokens.get(categoryToken);
+///          if (models === undefined) {
+///            models = new Set();
+///            categoriesToTtsModelTokens.set(categoryToken, models);
+///          }
+///          models.add(ttsModel);
+///        })
+///      });
+///    });
+///    setTtsModelsByCategoryToken(categoriesToTtsModelTokens);
 
-    // N * M with memoization should't be too bad here.
-    // Also note that the models should be lexographically sorted by name.
-    allTtsModels.forEach(ttsModel => {
-      if (ttsModel.category_tokens.length === 0) {
-        // TODO: Attach to "uncategorized" special category
-        return;
-      }
-      ttsModel.category_tokens.forEach(categoryToken => {
-        let ancestors = categoryTokenToAllAncestorTokens.get(categoryToken);
-        if (ancestors === undefined) {
-          ancestors = findAllAncestorTokens(categoryToken, categoriesByTokenMap);
-          categoryTokenToAllAncestorTokens.set(categoryToken, ancestors);
+    const nestedCategoryTokenToModelTokensMap : Map<string, Set<string>> = computedTtsCategoryAssignments?.category_token_to_tts_model_tokens.recursive || new Map();
+
+    let categoriesToTtsModelTokens = new Map();
+
+    nestedCategoryTokenToModelTokensMap.forEach((ttsModelTokens, categoryToken) => {
+      ttsModelTokens.forEach(ttsModelToken => {
+        let ttsModel = ttsModelsByTokenMap.get(ttsModelToken);
+        if (ttsModel === undefined) {
+          return;
         }
-        ancestors.forEach(categoryToken => {
-          let models : Set<TtsModelListItem> = categoriesToTtsModelTokens.get(categoryToken);
-          if (models === undefined) {
-            models = new Set();
-            categoriesToTtsModelTokens.set(categoryToken, models);
-          }
-          models.add(ttsModel);
-        })
-      });
-    });
+        let models : Set<TtsModelListItem> = categoriesToTtsModelTokens.get(categoryToken);
+        if (models === undefined) {
+          models = new Set();
+          categoriesToTtsModelTokens.set(categoryToken, models);
+        }
+        models.add(ttsModel);
+      })
+    })
+
     setTtsModelsByCategoryToken(categoriesToTtsModelTokens);
 
-  }, [allTtsCategories, allTtsModels]);
+  }, [allTtsCategories, allTtsModels, computedTtsCategoryAssignments]);
 
   return (
     <App 
@@ -133,6 +159,9 @@ export function AppWrapper(props: Props) {
         allTtsModels={allTtsModels}
         setAllTtsModels={setAllTtsModels}
 
+        computedTtsCategoryAssignments={computedTtsCategoryAssignments}
+        setComputedTtsCategoryAssignments={setComputedTtsCategoryAssignments}
+
         allTtsCategoriesByTokenMap={allCategoriesByTokenMap}
         allTtsModelsByTokenMap={allTtsModelsByTokenMap}
         ttsModelsByCategoryToken={ttsModelsByCategoryToken}
@@ -148,21 +177,21 @@ export function AppWrapper(props: Props) {
   )
 }
 
-function findAllAncestorTokens(categoryToken: string, allCategoriesByTokenMap: Map<string, TtsCategory>): Set<string> {
-  const ancestorTokens = recursiveFindAllAncestorTokens(categoryToken, allCategoriesByTokenMap);
-  return new Set(ancestorTokens);
-}
+/// function findAllAncestorTokens(categoryToken: string, allCategoriesByTokenMap: Map<string, TtsCategory>): Set<string> {
+///   const ancestorTokens = recursiveFindAllAncestorTokens(categoryToken, allCategoriesByTokenMap);
+///   return new Set(ancestorTokens);
+/// }
 
-function recursiveFindAllAncestorTokens(categoryToken: string, allCategoriesByTokenMap: Map<string, TtsCategory>): string[] {
-  let category = allCategoriesByTokenMap.get(categoryToken)
-  if (category === undefined) {
-    return [];
-  }
-  if (!category.maybe_super_category_token) {
-    return [categoryToken];
-  }
-  return [
-    ...recursiveFindAllAncestorTokens(category.maybe_super_category_token, allCategoriesByTokenMap), 
-    categoryToken,
-  ];
-}
+/// function recursiveFindAllAncestorTokens(categoryToken: string, allCategoriesByTokenMap: Map<string, TtsCategory>): string[] {
+///   let category = allCategoriesByTokenMap.get(categoryToken)
+///   if (category === undefined) {
+///     return [];
+///   }
+///   if (!category.maybe_super_category_token) {
+///     return [categoryToken];
+///   }
+///   return [
+///     ...recursiveFindAllAncestorTokens(category.maybe_super_category_token, allCategoriesByTokenMap), 
+///     categoryToken,
+///   ];
+/// }

packages/fakeyou.com/src/v2/api/category/GetComputedTtsCategoryAssignments.ts

@@ -0,0 +1,75 @@
+import { ApiConfig } from "@storyteller/components";
+
+export interface GetComputedTtsCategoryAssignmentsSuccessResponse {
+  success: boolean,
+  // NB: Not in use yet.
+  //utilized_tts_category_tokens: undefined,
+  category_token_to_tts_model_tokens: ModelTokensByCategoryToken,
+}
+
+export interface UtilizedCategoryTokens {
+  recursive: Set<string>,
+  leaf_only: Set<string>,
+}
+
+export interface ModelTokensByCategoryToken {
+  recursive: Map<string, Set<string>>,
+  leaf_only: Map<string, Set<string>>,
+}
+
+export interface GetComputedTtsCategoryAssignmentsErrorResponse {
+  success: boolean,
+}
+
+type GetComputedTtsCategoryAssignmentsResponse = GetComputedTtsCategoryAssignmentsSuccessResponse | GetComputedTtsCategoryAssignmentsErrorResponse;
+
+export function GetComputedTtsCategoryAssignmentsIsOk(response: GetComputedTtsCategoryAssignmentsResponse): response is GetComputedTtsCategoryAssignmentsSuccessResponse {
+  return response?.success === true;
+}
+
+export function GetComputedTtsCategoryAssignmentsIsError(response: GetComputedTtsCategoryAssignmentsResponse): response is GetComputedTtsCategoryAssignmentsErrorResponse {
+  return response?.success === false;
+}
+
+export async function GetComputedTtsCategoryAssignments() : Promise<GetComputedTtsCategoryAssignmentsResponse> 
+{
+  const endpoint = new ApiConfig().getComputedTtsCategoryAssignments();
+
+  console.error('GetComputedTtsCategoryAssignments')
+  
+  return await fetch(endpoint, {
+    method: 'GET',
+    headers: {
+      'Accept': 'application/json',
+    },
+    credentials: 'include',
+  })
+  .then(res => res.json())
+  .then(res => {
+    if (!res) {
+      return { success : false }; // TODO: This loses error semantics and is deprecated
+    }
+
+    if (GetComputedTtsCategoryAssignmentsIsOk(res)) {
+      // NB: Cast to appropriate types.
+      if (res.category_token_to_tts_model_tokens.recursive !== undefined) {
+        let entries : [string, Set<string>][] = Object.entries(res.category_token_to_tts_model_tokens.recursive).map(([category_token, model_tokens]) => {
+            return [category_token, new Set(model_tokens)];
+        });
+        res.category_token_to_tts_model_tokens.recursive = new Map(entries);
+      }
+      if (res.category_token_to_tts_model_tokens.leaf_only !== undefined) {
+        let entries : [string, Set<string>][] = Object.entries(res.category_token_to_tts_model_tokens.leaf_only).map(([category_token, model_tokens]) => {
+            return [category_token, new Set(model_tokens)];
+        });
+        res.category_token_to_tts_model_tokens.leaf_only = new Map(entries);
+      }
+      return res;
+    } else {
+      return { success : false };
+    }
+  })
+  .catch(e => {
+    return { success : false };
+  });
+}

packages/fakeyou.com/src/v2/view/PageContainer.tsx

@@ -62,6 +62,7 @@ import { CheckoutSuccessPage } from "./premium/CheckoutSuccessPage";
 import { CheckoutCancelPage } from "./premium/CheckoutCancelPage";
 import { PortalSuccessPage } from "./premium/PortalSuccessPage";
 import { PrivacyPage } from "./about/privacy_page/PrivacyPage";
+import { GetComputedTtsCategoryAssignmentsSuccessResponse } from "../api/category/GetComputedTtsCategoryAssignments";
 //import { LandingPage } from "./landing/LandingPage";
 //import { VcModelListPage } from "./vc/vc_model_list/VcModelListPage";
 
@@ -113,6 +114,9 @@ interface Props {
 
   allTtsCategories: TtsCategoryType[];
   setAllTtsCategories: (allTtsCategories: TtsCategoryType[]) => void;
+  
+  computedTtsCategoryAssignments?: GetComputedTtsCategoryAssignmentsSuccessResponse;
+  setComputedTtsCategoryAssignments: (categoryAssignments: GetComputedTtsCategoryAssignmentsSuccessResponse) => void;
 
   allTtsCategoriesByTokenMap: Map<string, TtsCategoryType>;
   allTtsModelsByTokenMap: Map<string, TtsModelListItem>;
@@ -474,6 +478,8 @@ class PageContainer extends React.Component<Props, State> {
                 allTtsCategoriesByTokenMap={
                   this.props.allTtsCategoriesByTokenMap
                 }
+                computedTtsCategoryAssignments={this.props.computedTtsCategoryAssignments}
+                setComputedTtsCategoryAssignments={this.props.setComputedTtsCategoryAssignments}
                 allTtsModelsByTokenMap={this.props.allTtsModelsByTokenMap}
                 ttsModelsByCategoryToken={this.props.ttsModelsByCategoryToken}
                 dropdownCategories={this.props.dropdownCategories}

packages/fakeyou.com/src/v2/view/tts/tts_model_list/TtsModelListPage.tsx

@@ -49,6 +49,7 @@ import { container, panel } from "../../../../data/animation";
 import { SessionSubscriptionsWrapper } from "@storyteller/components/src/session/SessionSubscriptionsWrapper";
 import { TtsPageHero } from "./TtsPageHero";
 import { Analytics } from "../../../../common/Analytics";
+import { GetComputedTtsCategoryAssignments, GetComputedTtsCategoryAssignmentsIsError, GetComputedTtsCategoryAssignmentsIsOk, GetComputedTtsCategoryAssignmentsSuccessResponse } from "../../../api/category/GetComputedTtsCategoryAssignments";
 
 export interface EnqueueJobResponsePayload {
   success: boolean;
@@ -90,6 +91,9 @@ interface Props {
   allTtsCategories: TtsCategoryType[];
   setAllTtsCategories: (allTtsCategories: TtsCategoryType[]) => void;
 
+  computedTtsCategoryAssignments?: GetComputedTtsCategoryAssignmentsSuccessResponse;
+  setComputedTtsCategoryAssignments: (categoryAssignments: GetComputedTtsCategoryAssignmentsSuccessResponse) => void;
+
   allTtsCategoriesByTokenMap: Map<string, TtsCategoryType>;
   allTtsModelsByTokenMap: Map<string, TtsModelListItem>;
   ttsModelsByCategoryToken: Map<string, Set<TtsModelListItem>>;
@@ -107,8 +111,10 @@ function TtsModelListPage(props: Props) {
   let {
     setTtsModels,
     setAllTtsCategories,
+    setComputedTtsCategoryAssignments,
     ttsModels,
     allTtsCategories,
+    computedTtsCategoryAssignments,
     maybeSelectedTtsModel,
     setMaybeSelectedTtsModel,
   } = props;
@@ -119,6 +125,9 @@ function TtsModelListPage(props: Props) {
 
   const ttsModelsLoaded = ttsModels.length > 0;
   const ttsCategoriesLoaded = allTtsCategories.length > 0;
+  const computedTtsCategoryAssignmentsLoaded = 
+    computedTtsCategoryAssignments !== undefined && 
+    computedTtsCategoryAssignments.category_token_to_tts_model_tokens.recursive.size > 0;
 
   const listModels = useCallback(async () => {
     if (ttsModelsLoaded) {
@@ -162,10 +171,23 @@ function TtsModelListPage(props: Props) {
     }
   }, [setAllTtsCategories, ttsCategoriesLoaded]);
 
+  const getComputedAssignments = useCallback(async () => {
+    if (computedTtsCategoryAssignmentsLoaded) {
+      return; // Already queried.
+    }
+    const computedAssignments = await GetComputedTtsCategoryAssignments();
+    if (GetComputedTtsCategoryAssignmentsIsOk(computedAssignments)) {
+      setComputedTtsCategoryAssignments(computedAssignments);
+    } else if (GetComputedTtsCategoryAssignmentsIsError(computedAssignments)) {
+      // TODO: Retry on decay function
+    }
+  }, [setComputedTtsCategoryAssignments, computedTtsCategoryAssignmentsLoaded]);
+
   useEffect(() => {
     listModels();
     listTtsCategories();
-  }, [listModels, listTtsCategories]);
+    getComputedAssignments()
+  }, [listModels, listTtsCategories, getComputedAssignments]);
 
   const handleChangeText = (ev: React.FormEvent<HTMLTextAreaElement>) => {
     const textValue = (ev.target as HTMLTextAreaElement).value;