Api2.5/list tts v2v models (#498)

this goes after usebackgroundmedia implementation
https://github.com/storytold/GottaGoFast/pull/497

with this (and the previous, the entire workflow of making a tts/v2v is:
- loading inference > previously done
- listing weights > now done
- requesting a tts > previously done
- request a v2v > 1 of the endpoint that's still missing in api2.5
- loading the new audio > implicitly done

---------

Co-authored-by: wil <wil@storyteller.ai>

app/Classes/ApiManager/WeightsApi.ts

@@ -31,6 +31,9 @@ export enum ScopedWeightCategory {
 interface WeightsRequest {
   username?: string;
   pageSize?: number;
+  weightType?: ScopedWeightType[];
+  weightCategory?: ScopedWeightCategory[];
+  //TODO: what is scoped??
   scopedWeightType?: ScopedWeightType[];
   scopedWeightCategory?: ScopedWeightCategory[];
 }

app/api.tsx

@@ -2,28 +2,28 @@ export const apiHost = "https://api.storyteller.ai";
 
 // Authentications
 // const createAccount = `${apiHost}}/create_account`;
-export const login = `${apiHost}/v1/login`;
-export const getSession = `${apiHost}/v1/session`;
-export const logout = `${apiHost}/v1/logout`;
+// export const login = `${apiHost}/v1/login`;
+// export const getSession = `${apiHost}/v1/session`;
+// export const logout = `${apiHost}/v1/logout`;
 
 // Inference Jobs
-export const getRecentJobs = `${apiHost}/v1/jobs/session`;
-export const getActiveJobs = `${apiHost}/v1/jobs/session?exclude_states=complete_success,complete_failure,dead,cancelled_by_user,cancelled_by_system`;
-export const getJobStatus = (jobToken: string) =>
-  `${apiHost}/v1/model_inference/job_status/${jobToken}`;
+// export const getRecentJobs = `${apiHost}/v1/jobs/session`;
+// export const getActiveJobs = `${apiHost}/v1/jobs/session?exclude_states=complete_success,complete_failure,dead,cancelled_by_user,cancelled_by_system`;
+// export const getJobStatus = (jobToken: string) =>
+// `${apiHost}/v1/model_inference/job_status/${jobToken}`;
 
 // TTS
-export const inferTts = `${apiHost}/tts/inference`;
-export const listTts = `${apiHost}/tts/list`;
+// export const inferTts = `${apiHost}/tts/inference`;
+// export const listTts = `${apiHost}/tts/list`;
 
 //V2V
-export const listV2V = `${apiHost}/v1/voice_conversion/model_list`;
+// export const listV2V = `${apiHost}/v1/voice_conversion/model_list`;
 export const inferV2V = `${apiHost}/v1/voice_conversion/inference`;
 export const uploadAudioV2V = `${apiHost}/v1/media_uploads/upload_audio`;
 
 // User Media
-export const listMediaByUser = (username: string) =>
-  `${apiHost}/v1/media_files/list/user/${username}`;
+// export const listMediaByUser = (username: string) =>
+//   `${apiHost}/v1/media_files/list/user/${username}`;
 
 // Get Any Media
 export const getMediaFileByToken = (fileToken: string) =>

app/pages/PageEnigma/Editor/api_manager.ts

@@ -1,7 +1,6 @@
 import { v4 as uuidv4 } from "uuid";
 import * as THREE from "three";
 import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js";
-import { STORAGE_KEYS } from "~/enums";
 import {
   environmentVariables,
   signalScene,

app/pages/PageEnigma/comps/GenerateModals/listMoviesByUser.tsx

@@ -1,27 +0,0 @@
-import { listMediaByUser } from "~/api";
-import { MediaInfo } from "~/pages/PageEnigma/models/movies";
-
-export const listMoviesByUser = async (
-  username: string,
-): Promise<MediaInfo[]> => {
-  return fetch(
-    listMediaByUser(username) +
-      "?" +
-      new URLSearchParams({ filter_media_type: "video" }),
-    {
-      method: "GET",
-      headers: {
-        Accept: "application/json",
-      },
-      // credentials: 'include'
-    },
-  )
-    .then((res) => res.json())
-    .then((res) => {
-      if (res.success && res.results) {
-        return res.results;
-      }
-      Promise.reject();
-    })
-    .catch(() => ({ success: false }));
-};

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/AudioTab.tsx

@@ -3,12 +3,8 @@ import { useSignals } from "@preact/signals-react/runtime";
 
 import { AudioTabPages } from "~/pages/PageEnigma/enums";
 import { AudioPanelState } from "~/pages/PageEnigma/models/voice";
-import {
-  TtsModelListItem,
-  VoiceConversionModelListItem,
-} from "~/pages/PageEnigma/models";
 
-import { ListTtsModels, ListVoiceConversionModels } from "./utilities";
+import { Weight } from "~/models";
 import { initialTtsState, initialV2VState } from "./values";
 
 import { PageAudioLibrary } from "./pageAudioLibrary";
@@ -16,6 +12,12 @@ import { PageAudioGeneration } from "./pageAudioGeneration";
 import { PageSelectTtsModel } from "./pageSelectTtsModel";
 import { PageSelectV2VModel } from "./pageSelectV2VModel";
 import { PollUserAudioItems } from "~/hooks/useBackgroundLoadingMedia/utilities";
+import {
+  ScopedWeightCategory,
+  WeightsApi,
+} from "~/Classes/ApiManager/WeightsApi";
+import { addToast } from "~/signals";
+import { ToastTypes } from "~/enums";
 
 export const AudioTab = () => {
   useSignals();
@@ -30,24 +32,50 @@ export const AudioTab = () => {
     v2vState: initialV2VState,
   });
 
-  const [ttsModels, setTtsModels] = useState<Array<TtsModelListItem>>([]);
-  const [v2vModels, setV2VModels] = useState<
-    Array<VoiceConversionModelListItem>
-  >([]);
+  const [ttsModels, setTtsModels] = useState<Weight[]>([]);
+  const [v2vModels, setV2VModels] = useState<Weight[]>([]);
+
+  const ListTtsModels = useCallback(async () => {
+    const weightApi = new WeightsApi();
+    const response = await weightApi.ListWeights({
+      weightCategory: [ScopedWeightCategory.TEXT_TO_SPEECH],
+      pageSize: 10000,
+    });
+    if (response.success && response.data) {
+      setTtsModels(response.data);
+      return;
+    }
+    addToast(
+      ToastTypes.ERROR,
+      response.errorMessage || "Unknown Error in Getting TTS Models",
+    );
+  }, []);
+
+  const ListVoiceConversionModels = useCallback(async () => {
+    const weightApi = new WeightsApi();
+    const response = await weightApi.ListWeights({
+      weightCategory: [ScopedWeightCategory.VOICE_CONVERSION],
+      pageSize: 10000,
+    });
+    if (response.success && response.data) {
+      setV2VModels(response.data);
+      return;
+    }
+    addToast(
+      ToastTypes.ERROR,
+      response.errorMessage || "Unknown Error in Getting V2V Models",
+    );
+  }, []);
 
   useEffect(() => {
     if (!state.firstLoad) {
       //fetch all the data on first load once, after securing auth info
-      ListTtsModels().then((res) => {
-        if (res) setTtsModels(res);
-      });
-      ListVoiceConversionModels().then((res) => {
-        if (res) setV2VModels(res);
-      });
+      ListTtsModels();
+      ListVoiceConversionModels();
       setState((curr) => ({ ...curr, firstLoad: true }));
       // completed the first load
     }
-  }, [state]);
+  }, [state, ListTtsModels, ListVoiceConversionModels]);
 
   const changePage = useCallback((newPage: AudioTabPages) => {
     setState((curr) => ({

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectTtsModel.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useState } from "react";
 import { Input } from "~/components";
-import { TtsModelListItem } from "~/pages/PageEnigma/models/tts";
+import { Weight } from "~/models";
 import { TabTitle } from "~/pages/PageEnigma/comps/SidePanelTabs/comps/TabTitle";
 import { VoiceModelElement } from "./voiceModelElement";
 import { AudioTabPages } from "~/pages/PageEnigma/enums";
@@ -11,8 +11,8 @@ export const PageSelectTtsModel = ({
   onSelect,
 }: {
   changePage: (newPage: AudioTabPages) => void;
-  ttsModels: Array<TtsModelListItem>;
-  onSelect: (item: TtsModelListItem) => void;
+  ttsModels: Weight[];
+  onSelect: (item: Weight) => void;
 }) => {
   const [query, setQuery] = useState("");
   const filteredListOfModels =
@@ -50,9 +50,9 @@ export const PageSelectTtsModel = ({
           {slicedArray.map((item) => {
             return (
               <VoiceModelElement
-                key={item.model_token}
+                key={item.weight_token}
                 model={item}
-                onSelect={(item) => onSelect(item as TtsModelListItem)}
+                onSelect={(item) => onSelect(item)}
               />
             );
           })}

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectV2VModel.tsx

@@ -1,9 +1,9 @@
 import { useCallback, useState } from "react";
 
+import { Weight } from "~/models";
 import { Input } from "~/components";
 import { TabTitle } from "~/pages/PageEnigma/comps/SidePanelTabs/comps/TabTitle";
 import { VoiceModelElement } from "./voiceModelElement";
-import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models";
 import { AudioTabPages } from "~/pages/PageEnigma/enums";
 
 export const PageSelectV2VModel = ({
@@ -12,8 +12,8 @@ export const PageSelectV2VModel = ({
   onSelect,
 }: {
   changePage: (newPage: AudioTabPages) => void;
-  v2vModels: Array<VoiceConversionModelListItem>;
-  onSelect: (item: VoiceConversionModelListItem) => void;
+  v2vModels: Weight[];
+  onSelect: (item: Weight) => void;
 }) => {
   const [query, setQuery] = useState("");
   const filteredListOfModels =
@@ -50,11 +50,9 @@ export const PageSelectV2VModel = ({
         {slicedArray.map((item) => {
           return (
             <VoiceModelElement
-              key={item.token}
+              key={item.weight_token}
               model={item}
-              onSelect={(item) =>
-                onSelect(item as VoiceConversionModelListItem)
-              }
+              onSelect={(item) => onSelect(item)}
             />
           );
         })}

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageTTS.tsx

@@ -22,7 +22,7 @@ export const PageTTS = ({
   setTtsState: (newState: TtsState) => void;
 }) => {
   const requestTts = useCallback(async () => {
-    const modelToken = ttsState.voice ? ttsState.voice.model_token : undefined;
+    const modelToken = ttsState.voice ? ttsState.voice.weight_token : undefined;
 
     if (!modelToken) {
       addToast(ToastTypes.ERROR, "Please first pick a voice");
@@ -72,7 +72,7 @@ export const PageTTS = ({
             <>
               <H4>{ttsState.voice.title}</H4>
               <H6 className="text-white/70">
-                by {ttsState.voice.creator_display_name}
+                by {ttsState.voice.creator.display_name}
               </H6>
             </>
           )}

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageVoiceToVoice.tsx

@@ -13,7 +13,8 @@ import { GenerateVoiceConversion } from "./utilities";
 import { H4, H6, Button, Label, UploadAudioComponent } from "~/components";
 
 import { AudioTabPages } from "~/pages/PageEnigma/enums";
-import { startPollingActiveJobs } from "~/signals";
+import { addToast, startPollingActiveJobs } from "~/signals";
+import { ToastTypes } from "~/enums";
 
 export const PageVoicetoVoice = ({
   changePage,
@@ -25,23 +26,35 @@ export const PageVoicetoVoice = ({
   setV2VState: (newState: V2VState) => void;
 }) => {
   const requestV2V = useCallback(() => {
-    const modelToken = v2vState.voice ? v2vState.voice.token : undefined;
-    if (modelToken && v2vState.inputFileToken) {
-      const request: EnqueueVoiceConversionRequest = {
-        uuid_idempotency_token: uuidv4(),
-        voice_conversion_model_token: modelToken,
-        source_media_upload_token: v2vState.inputFileToken,
-      };
-
-      GenerateVoiceConversion(request).then(
-        (res: EnqueueVoiceConversionResponse) => {
-          if (res && res.inference_job_token) {
-            startPollingActiveJobs();
-            changePage(AudioTabPages.LIBRARY);
-          }
-        },
-      );
+    const modelToken = v2vState.voice ? v2vState.voice.weight_token : undefined;
+    if (!modelToken) {
+      addToast(ToastTypes.ERROR, "Please first pick a voice");
+      return;
     }
+    if (!v2vState.inputFileToken) {
+      addToast(ToastTypes.ERROR, "Please upload a voice clip to convert from");
+      return;
+    }
+    const request: EnqueueVoiceConversionRequest = {
+      uuid_idempotency_token: uuidv4(),
+      voice_conversion_model_token: modelToken,
+      source_media_upload_token: v2vState.inputFileToken,
+    };
+
+    GenerateVoiceConversion(request).then(
+      (res: EnqueueVoiceConversionResponse) => {
+        console.log(request, res);
+        if (res && res.inference_job_token) {
+          startPollingActiveJobs();
+          changePage(AudioTabPages.LIBRARY);
+          return;
+        }
+        addToast(
+          ToastTypes.ERROR,
+          res.error_reason || "Unknown Error in Generating Voice Conversions",
+        );
+      },
+    );
   }, [v2vState, changePage]);
 
   return (

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/typesImported.ts

@@ -1,10 +1,3 @@
-import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models/types";
-
-export interface VoiceConversionModelListResponse {
-  success: boolean;
-  models: Array<VoiceConversionModelListItem>;
-}
-
 export interface EnqueueVoiceConversionRequest {
   uuid_idempotency_token: string;
   voice_conversion_model_token: string;
@@ -19,4 +12,5 @@ export interface EnqueueVoiceConversionRequest {
 export interface EnqueueVoiceConversionResponse {
   success: boolean;
   inference_job_token?: string;
+  error_reason?: string;
 }

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/utilities.ts

@@ -1,135 +1,10 @@
+import { getMediaFileByToken, inferV2V } from "~/api";
 import {
-  listTts,
-  listV2V,
-  inferTts,
-  listMediaByUser,
-  getMediaFileByToken,
-  inferV2V,
-} from "~/api";
-import {
-  VoiceConversionModelListResponse,
   EnqueueVoiceConversionRequest,
   EnqueueVoiceConversionResponse,
 } from "./typesImported";
-import {
-  TtsModelListItem,
-  TtsModelListResponsePayload,
-  StatusLike,
-  GenerateTtsAudioRequest,
-  GenerateTtsAudioResponse,
-} from "~/pages/PageEnigma/models/tts";
 
-import { GenerateTtsAudioErrorType } from "~/pages/PageEnigma/enums";
-import {
-  GetMediaFileResponse,
-  VoiceConversionModelListItem,
-} from "~/pages/PageEnigma/models";
-
-import { authentication } from "~/signals";
-const { userInfo } = authentication;
-
-export const ListAudioByUser = () => {
-  if (userInfo.value) {
-    return fetch(
-      listMediaByUser(userInfo.value.username) + "?filter_media_type=audio",
-      {
-        method: "GET",
-        headers: {
-          Accept: "application/json",
-        },
-        credentials: "include",
-      },
-    )
-      .then((res) => res.json())
-      .then((res) => {
-        if (res.success && res.results) {
-          return res.results;
-        } else {
-          Promise.reject();
-        }
-      })
-      .catch(() => ({ success: false }));
-  } else {
-    return new Promise((resolve) => {
-      resolve({ success: false });
-    });
-  }
-};
-
-export function ListTtsModels(): Promise<Array<TtsModelListItem> | undefined> {
-  if (userInfo.value) {
-    return fetch(listTts, {
-      method: "GET",
-      headers: {
-        Accept: "application/json",
-      },
-      credentials: "include",
-    })
-      .then((res) => res.json())
-      .then((res) => {
-        const response: TtsModelListResponsePayload = res;
-        if (!response.success) {
-          return undefined;
-        }
-        return response?.models;
-      })
-      .catch(() => {
-        return undefined;
-      });
-  } else {
-    return new Promise((resolve) => {
-      resolve(undefined);
-    });
-  }
-}
-
-export function maybeMapError(
-  statuslike: StatusLike,
-): GenerateTtsAudioErrorType | undefined {
-  switch (statuslike.status) {
-    case 400:
-      return GenerateTtsAudioErrorType.BadRequest;
-    case 404:
-      return GenerateTtsAudioErrorType.NotFound;
-    case 429:
-      return GenerateTtsAudioErrorType.TooManyRequests;
-    case 500:
-      return GenerateTtsAudioErrorType.ServerError;
-  }
-}
-
-export function GenerateTtsAudio(
-  request: GenerateTtsAudioRequest,
-): Promise<GenerateTtsAudioResponse> {
-  return fetch(inferTts, {
-    method: "POST",
-    headers: {
-      Accept: "application/json",
-      "Content-Type": "application/json",
-    },
-    credentials: "include",
-    body: JSON.stringify(request),
-  })
-    .then((res) => res.json())
-    .then((res) => {
-      if (!("inference_job_token" in res)) {
-        return { error: GenerateTtsAudioErrorType.UnknownError };
-      }
-      const ret: GenerateTtsAudioResponse = {
-        success: true,
-        inference_job_token: res.inference_job_token,
-        inference_job_token_type: res.inference_job_token_type,
-      };
-      return ret;
-    })
-    .catch((e) => {
-      const maybeError = maybeMapError(e);
-      if (maybeError !== undefined) {
-        return { error: maybeError };
-      }
-      return { error: GenerateTtsAudioErrorType.UnknownError };
-    });
-}
+import { GetMediaFileResponse } from "~/pages/PageEnigma/models";
 
 export function GetMediaFileByToken(
   fileToken: string,
@@ -164,30 +39,6 @@ export function GetMediaFileByToken(
     });
 }
 
-export function ListVoiceConversionModels(): Promise<
-  Array<VoiceConversionModelListItem> | undefined
-> {
-  return fetch(listV2V, {
-    method: "GET",
-    headers: {
-      Accept: "application/json",
-      "Content-Type": "application/json",
-    },
-    credentials: "include",
-  })
-    .then((res) => res.json())
-    .then((res) => {
-      const response: VoiceConversionModelListResponse = res;
-      if (!response.success) {
-        return;
-      }
-      return response?.models;
-    })
-    .catch(() => {
-      return undefined;
-    });
-}
-
 export function GenerateVoiceConversion(
   request: EnqueueVoiceConversionRequest,
 ): Promise<EnqueueVoiceConversionResponse> {
@@ -198,6 +49,7 @@ export function GenerateVoiceConversion(
       "Content-Type": "application/json",
     },
     body: JSON.stringify(request),
+    credentials: "include",
   })
     .then((res) => res.json())
     .then((res) => {

app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/voiceModelElement.tsx

@@ -1,24 +1,15 @@
-import { TtsModelListItem } from "~/pages/PageEnigma/models/tts";
+import { Weight } from "~/models";
 import { H4, H6 } from "~/components";
-import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models";
 
 export const VoiceModelElement = ({
   model,
   onSelect,
 }: {
-  model: TtsModelListItem | VoiceConversionModelListItem;
-  onSelect: (item: TtsModelListItem | VoiceConversionModelListItem) => void;
+  model: Weight;
+  onSelect: (item: Weight) => void;
 }) => {
-  let creatorName: string | undefined;
-  if ("creator_display_name" in model) {
-    //case of TTS Models
-    creatorName = model.creator_display_name;
-  } else if ("creator" in model) {
-    //case of V2V Models
-    creatorName = model.creator.display_name;
-  } else {
-    creatorName = undefined;
-  }
+  const creatorName = model.creator.display_name;
+
   return (
     <button
       className="flex cursor-pointer items-center justify-between gap-3 rounded-lg border-2 border-transparent bg-brand-secondary p-3 text-start transition-all hover:border-ui-controls-button hover:bg-ui-controls-button/40"

app/pages/PageEnigma/models/voice.ts

@@ -1,5 +1,4 @@
-import { TtsModelListItem } from "~/pages/PageEnigma/models";
-import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models/types";
+import { Weight } from "~/models";
 import { AudioTabPages } from "~/pages/PageEnigma/enums";
 
 export type AudioPanelState = {
@@ -10,12 +9,12 @@ export type AudioPanelState = {
   v2vState: V2VState;
 };
 export type TtsState = {
-  voice: TtsModelListItem | undefined;
+  voice: Weight | undefined;
   text: string;
 };
 
 export type V2VState = {
-  voice: VoiceConversionModelListItem | undefined;
+  voice: Weight | undefined;
   file: File | undefined;
   inputFileToken: string | undefined;
 };