From 5b228de14e241e901b19e03ccbbf45afc7ce52ba Mon Sep 17 00:00:00 2001 From: wil Date: Tue, 18 Jun 2024 23:16:40 -0400 Subject: [PATCH] Api2.5/list tts v2v models (#498) this goes after usebackgroundmedia implementation https://github.com/storytold/GottaGoFast/pull/497 with this (and the previous, the entire workflow of making a tts/v2v is: - loading inference > previously done - listing weights > now done - requesting a tts > previously done - request a v2v > 1 of the endpoint that's still missing in api2.5 - loading the new audio > implicitly done --------- Co-authored-by: wil --- app/Classes/ApiManager/WeightsApi.ts | 3 + app/api.tsx | 24 +-- app/pages/PageEnigma/Editor/api_manager.ts | 1 - .../comps/GenerateModals/listMoviesByUser.tsx | 27 --- .../itemTabs/AudioTab/AudioTab.tsx | 60 +++++-- .../itemTabs/AudioTab/pageSelectTtsModel.tsx | 10 +- .../itemTabs/AudioTab/pageSelectV2VModel.tsx | 12 +- .../itemTabs/AudioTab/pageTTS.tsx | 4 +- .../itemTabs/AudioTab/pageVoiceToVoice.tsx | 47 ++++-- .../itemTabs/AudioTab/typesImported.ts | 8 +- .../itemTabs/AudioTab/utilities.ts | 154 +----------------- .../itemTabs/AudioTab/voiceModelElement.tsx | 19 +-- app/pages/PageEnigma/models/voice.ts | 7 +- 13 files changed, 113 insertions(+), 263 deletions(-) delete mode 100644 app/pages/PageEnigma/comps/GenerateModals/listMoviesByUser.tsx diff --git a/app/Classes/ApiManager/WeightsApi.ts b/app/Classes/ApiManager/WeightsApi.ts index c18d4bcaa4..44f6b02adf 100644 --- a/app/Classes/ApiManager/WeightsApi.ts +++ b/app/Classes/ApiManager/WeightsApi.ts @@ -31,6 +31,9 @@ export enum ScopedWeightCategory { interface WeightsRequest { username?: string; pageSize?: number; + weightType?: ScopedWeightType[]; + weightCategory?: ScopedWeightCategory[]; + //TODO: what is scoped?? scopedWeightType?: ScopedWeightType[]; scopedWeightCategory?: ScopedWeightCategory[]; } diff --git a/app/api.tsx b/app/api.tsx index e039c5a3ed..7c10d19b90 100644 --- a/app/api.tsx +++ b/app/api.tsx @@ -2,28 +2,28 @@ export const apiHost = "https://api.storyteller.ai"; // Authentications // const createAccount = `${apiHost}}/create_account`; -export const login = `${apiHost}/v1/login`; -export const getSession = `${apiHost}/v1/session`; -export const logout = `${apiHost}/v1/logout`; +// export const login = `${apiHost}/v1/login`; +// export const getSession = `${apiHost}/v1/session`; +// export const logout = `${apiHost}/v1/logout`; // Inference Jobs -export const getRecentJobs = `${apiHost}/v1/jobs/session`; -export const getActiveJobs = `${apiHost}/v1/jobs/session?exclude_states=complete_success,complete_failure,dead,cancelled_by_user,cancelled_by_system`; -export const getJobStatus = (jobToken: string) => - `${apiHost}/v1/model_inference/job_status/${jobToken}`; +// export const getRecentJobs = `${apiHost}/v1/jobs/session`; +// export const getActiveJobs = `${apiHost}/v1/jobs/session?exclude_states=complete_success,complete_failure,dead,cancelled_by_user,cancelled_by_system`; +// export const getJobStatus = (jobToken: string) => +// `${apiHost}/v1/model_inference/job_status/${jobToken}`; // TTS -export const inferTts = `${apiHost}/tts/inference`; -export const listTts = `${apiHost}/tts/list`; +// export const inferTts = `${apiHost}/tts/inference`; +// export const listTts = `${apiHost}/tts/list`; //V2V -export const listV2V = `${apiHost}/v1/voice_conversion/model_list`; +// export const listV2V = `${apiHost}/v1/voice_conversion/model_list`; export const inferV2V = `${apiHost}/v1/voice_conversion/inference`; export const uploadAudioV2V = `${apiHost}/v1/media_uploads/upload_audio`; // User Media -export const listMediaByUser = (username: string) => - `${apiHost}/v1/media_files/list/user/${username}`; +// export const listMediaByUser = (username: string) => +// `${apiHost}/v1/media_files/list/user/${username}`; // Get Any Media export const getMediaFileByToken = (fileToken: string) => diff --git a/app/pages/PageEnigma/Editor/api_manager.ts b/app/pages/PageEnigma/Editor/api_manager.ts index fc4e80202c..6c7deb7468 100644 --- a/app/pages/PageEnigma/Editor/api_manager.ts +++ b/app/pages/PageEnigma/Editor/api_manager.ts @@ -1,7 +1,6 @@ import { v4 as uuidv4 } from "uuid"; import * as THREE from "three"; import { GLTFExporter } from "three/addons/exporters/GLTFExporter.js"; -import { STORAGE_KEYS } from "~/enums"; import { environmentVariables, signalScene, diff --git a/app/pages/PageEnigma/comps/GenerateModals/listMoviesByUser.tsx b/app/pages/PageEnigma/comps/GenerateModals/listMoviesByUser.tsx deleted file mode 100644 index 10b544d3c0..0000000000 --- a/app/pages/PageEnigma/comps/GenerateModals/listMoviesByUser.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import { listMediaByUser } from "~/api"; -import { MediaInfo } from "~/pages/PageEnigma/models/movies"; - -export const listMoviesByUser = async ( - username: string, -): Promise => { - return fetch( - listMediaByUser(username) + - "?" + - new URLSearchParams({ filter_media_type: "video" }), - { - method: "GET", - headers: { - Accept: "application/json", - }, - // credentials: 'include' - }, - ) - .then((res) => res.json()) - .then((res) => { - if (res.success && res.results) { - return res.results; - } - Promise.reject(); - }) - .catch(() => ({ success: false })); -}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/AudioTab.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/AudioTab.tsx index ee69f0dea4..5601a47cdf 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/AudioTab.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/AudioTab.tsx @@ -3,12 +3,8 @@ import { useSignals } from "@preact/signals-react/runtime"; import { AudioTabPages } from "~/pages/PageEnigma/enums"; import { AudioPanelState } from "~/pages/PageEnigma/models/voice"; -import { - TtsModelListItem, - VoiceConversionModelListItem, -} from "~/pages/PageEnigma/models"; -import { ListTtsModels, ListVoiceConversionModels } from "./utilities"; +import { Weight } from "~/models"; import { initialTtsState, initialV2VState } from "./values"; import { PageAudioLibrary } from "./pageAudioLibrary"; @@ -16,6 +12,12 @@ import { PageAudioGeneration } from "./pageAudioGeneration"; import { PageSelectTtsModel } from "./pageSelectTtsModel"; import { PageSelectV2VModel } from "./pageSelectV2VModel"; import { PollUserAudioItems } from "~/hooks/useBackgroundLoadingMedia/utilities"; +import { + ScopedWeightCategory, + WeightsApi, +} from "~/Classes/ApiManager/WeightsApi"; +import { addToast } from "~/signals"; +import { ToastTypes } from "~/enums"; export const AudioTab = () => { useSignals(); @@ -30,24 +32,50 @@ export const AudioTab = () => { v2vState: initialV2VState, }); - const [ttsModels, setTtsModels] = useState>([]); - const [v2vModels, setV2VModels] = useState< - Array - >([]); + const [ttsModels, setTtsModels] = useState([]); + const [v2vModels, setV2VModels] = useState([]); + + const ListTtsModels = useCallback(async () => { + const weightApi = new WeightsApi(); + const response = await weightApi.ListWeights({ + weightCategory: [ScopedWeightCategory.TEXT_TO_SPEECH], + pageSize: 10000, + }); + if (response.success && response.data) { + setTtsModels(response.data); + return; + } + addToast( + ToastTypes.ERROR, + response.errorMessage || "Unknown Error in Getting TTS Models", + ); + }, []); + + const ListVoiceConversionModels = useCallback(async () => { + const weightApi = new WeightsApi(); + const response = await weightApi.ListWeights({ + weightCategory: [ScopedWeightCategory.VOICE_CONVERSION], + pageSize: 10000, + }); + if (response.success && response.data) { + setV2VModels(response.data); + return; + } + addToast( + ToastTypes.ERROR, + response.errorMessage || "Unknown Error in Getting V2V Models", + ); + }, []); useEffect(() => { if (!state.firstLoad) { //fetch all the data on first load once, after securing auth info - ListTtsModels().then((res) => { - if (res) setTtsModels(res); - }); - ListVoiceConversionModels().then((res) => { - if (res) setV2VModels(res); - }); + ListTtsModels(); + ListVoiceConversionModels(); setState((curr) => ({ ...curr, firstLoad: true })); // completed the first load } - }, [state]); + }, [state, ListTtsModels, ListVoiceConversionModels]); const changePage = useCallback((newPage: AudioTabPages) => { setState((curr) => ({ diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectTtsModel.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectTtsModel.tsx index d175b22276..58ae9e400c 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectTtsModel.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectTtsModel.tsx @@ -1,6 +1,6 @@ import { useCallback, useState } from "react"; import { Input } from "~/components"; -import { TtsModelListItem } from "~/pages/PageEnigma/models/tts"; +import { Weight } from "~/models"; import { TabTitle } from "~/pages/PageEnigma/comps/SidePanelTabs/comps/TabTitle"; import { VoiceModelElement } from "./voiceModelElement"; import { AudioTabPages } from "~/pages/PageEnigma/enums"; @@ -11,8 +11,8 @@ export const PageSelectTtsModel = ({ onSelect, }: { changePage: (newPage: AudioTabPages) => void; - ttsModels: Array; - onSelect: (item: TtsModelListItem) => void; + ttsModels: Weight[]; + onSelect: (item: Weight) => void; }) => { const [query, setQuery] = useState(""); const filteredListOfModels = @@ -50,9 +50,9 @@ export const PageSelectTtsModel = ({ {slicedArray.map((item) => { return ( onSelect(item as TtsModelListItem)} + onSelect={(item) => onSelect(item)} /> ); })} diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectV2VModel.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectV2VModel.tsx index 5de19cd153..70f0633789 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectV2VModel.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageSelectV2VModel.tsx @@ -1,9 +1,9 @@ import { useCallback, useState } from "react"; +import { Weight } from "~/models"; import { Input } from "~/components"; import { TabTitle } from "~/pages/PageEnigma/comps/SidePanelTabs/comps/TabTitle"; import { VoiceModelElement } from "./voiceModelElement"; -import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models"; import { AudioTabPages } from "~/pages/PageEnigma/enums"; export const PageSelectV2VModel = ({ @@ -12,8 +12,8 @@ export const PageSelectV2VModel = ({ onSelect, }: { changePage: (newPage: AudioTabPages) => void; - v2vModels: Array; - onSelect: (item: VoiceConversionModelListItem) => void; + v2vModels: Weight[]; + onSelect: (item: Weight) => void; }) => { const [query, setQuery] = useState(""); const filteredListOfModels = @@ -50,11 +50,9 @@ export const PageSelectV2VModel = ({ {slicedArray.map((item) => { return ( - onSelect(item as VoiceConversionModelListItem) - } + onSelect={(item) => onSelect(item)} /> ); })} diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageTTS.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageTTS.tsx index 933fdab7a4..cce7f03597 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageTTS.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageTTS.tsx @@ -22,7 +22,7 @@ export const PageTTS = ({ setTtsState: (newState: TtsState) => void; }) => { const requestTts = useCallback(async () => { - const modelToken = ttsState.voice ? ttsState.voice.model_token : undefined; + const modelToken = ttsState.voice ? ttsState.voice.weight_token : undefined; if (!modelToken) { addToast(ToastTypes.ERROR, "Please first pick a voice"); @@ -72,7 +72,7 @@ export const PageTTS = ({ <>

{ttsState.voice.title}

- by {ttsState.voice.creator_display_name} + by {ttsState.voice.creator.display_name}
)} diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageVoiceToVoice.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageVoiceToVoice.tsx index 9e6937b57f..f4410b7bcf 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageVoiceToVoice.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/pageVoiceToVoice.tsx @@ -13,7 +13,8 @@ import { GenerateVoiceConversion } from "./utilities"; import { H4, H6, Button, Label, UploadAudioComponent } from "~/components"; import { AudioTabPages } from "~/pages/PageEnigma/enums"; -import { startPollingActiveJobs } from "~/signals"; +import { addToast, startPollingActiveJobs } from "~/signals"; +import { ToastTypes } from "~/enums"; export const PageVoicetoVoice = ({ changePage, @@ -25,23 +26,35 @@ export const PageVoicetoVoice = ({ setV2VState: (newState: V2VState) => void; }) => { const requestV2V = useCallback(() => { - const modelToken = v2vState.voice ? v2vState.voice.token : undefined; - if (modelToken && v2vState.inputFileToken) { - const request: EnqueueVoiceConversionRequest = { - uuid_idempotency_token: uuidv4(), - voice_conversion_model_token: modelToken, - source_media_upload_token: v2vState.inputFileToken, - }; - - GenerateVoiceConversion(request).then( - (res: EnqueueVoiceConversionResponse) => { - if (res && res.inference_job_token) { - startPollingActiveJobs(); - changePage(AudioTabPages.LIBRARY); - } - }, - ); + const modelToken = v2vState.voice ? v2vState.voice.weight_token : undefined; + if (!modelToken) { + addToast(ToastTypes.ERROR, "Please first pick a voice"); + return; } + if (!v2vState.inputFileToken) { + addToast(ToastTypes.ERROR, "Please upload a voice clip to convert from"); + return; + } + const request: EnqueueVoiceConversionRequest = { + uuid_idempotency_token: uuidv4(), + voice_conversion_model_token: modelToken, + source_media_upload_token: v2vState.inputFileToken, + }; + + GenerateVoiceConversion(request).then( + (res: EnqueueVoiceConversionResponse) => { + console.log(request, res); + if (res && res.inference_job_token) { + startPollingActiveJobs(); + changePage(AudioTabPages.LIBRARY); + return; + } + addToast( + ToastTypes.ERROR, + res.error_reason || "Unknown Error in Generating Voice Conversions", + ); + }, + ); }, [v2vState, changePage]); return ( diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/typesImported.ts b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/typesImported.ts index 951a25c86b..3b6ef86c0c 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/typesImported.ts +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/typesImported.ts @@ -1,10 +1,3 @@ -import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models/types"; - -export interface VoiceConversionModelListResponse { - success: boolean; - models: Array; -} - export interface EnqueueVoiceConversionRequest { uuid_idempotency_token: string; voice_conversion_model_token: string; @@ -19,4 +12,5 @@ export interface EnqueueVoiceConversionRequest { export interface EnqueueVoiceConversionResponse { success: boolean; inference_job_token?: string; + error_reason?: string; } diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/utilities.ts b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/utilities.ts index a8ad3449c7..5d95cbebea 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/utilities.ts +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/utilities.ts @@ -1,135 +1,10 @@ +import { getMediaFileByToken, inferV2V } from "~/api"; import { - listTts, - listV2V, - inferTts, - listMediaByUser, - getMediaFileByToken, - inferV2V, -} from "~/api"; -import { - VoiceConversionModelListResponse, EnqueueVoiceConversionRequest, EnqueueVoiceConversionResponse, } from "./typesImported"; -import { - TtsModelListItem, - TtsModelListResponsePayload, - StatusLike, - GenerateTtsAudioRequest, - GenerateTtsAudioResponse, -} from "~/pages/PageEnigma/models/tts"; -import { GenerateTtsAudioErrorType } from "~/pages/PageEnigma/enums"; -import { - GetMediaFileResponse, - VoiceConversionModelListItem, -} from "~/pages/PageEnigma/models"; - -import { authentication } from "~/signals"; -const { userInfo } = authentication; - -export const ListAudioByUser = () => { - if (userInfo.value) { - return fetch( - listMediaByUser(userInfo.value.username) + "?filter_media_type=audio", - { - method: "GET", - headers: { - Accept: "application/json", - }, - credentials: "include", - }, - ) - .then((res) => res.json()) - .then((res) => { - if (res.success && res.results) { - return res.results; - } else { - Promise.reject(); - } - }) - .catch(() => ({ success: false })); - } else { - return new Promise((resolve) => { - resolve({ success: false }); - }); - } -}; - -export function ListTtsModels(): Promise | undefined> { - if (userInfo.value) { - return fetch(listTts, { - method: "GET", - headers: { - Accept: "application/json", - }, - credentials: "include", - }) - .then((res) => res.json()) - .then((res) => { - const response: TtsModelListResponsePayload = res; - if (!response.success) { - return undefined; - } - return response?.models; - }) - .catch(() => { - return undefined; - }); - } else { - return new Promise((resolve) => { - resolve(undefined); - }); - } -} - -export function maybeMapError( - statuslike: StatusLike, -): GenerateTtsAudioErrorType | undefined { - switch (statuslike.status) { - case 400: - return GenerateTtsAudioErrorType.BadRequest; - case 404: - return GenerateTtsAudioErrorType.NotFound; - case 429: - return GenerateTtsAudioErrorType.TooManyRequests; - case 500: - return GenerateTtsAudioErrorType.ServerError; - } -} - -export function GenerateTtsAudio( - request: GenerateTtsAudioRequest, -): Promise { - return fetch(inferTts, { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - }, - credentials: "include", - body: JSON.stringify(request), - }) - .then((res) => res.json()) - .then((res) => { - if (!("inference_job_token" in res)) { - return { error: GenerateTtsAudioErrorType.UnknownError }; - } - const ret: GenerateTtsAudioResponse = { - success: true, - inference_job_token: res.inference_job_token, - inference_job_token_type: res.inference_job_token_type, - }; - return ret; - }) - .catch((e) => { - const maybeError = maybeMapError(e); - if (maybeError !== undefined) { - return { error: maybeError }; - } - return { error: GenerateTtsAudioErrorType.UnknownError }; - }); -} +import { GetMediaFileResponse } from "~/pages/PageEnigma/models"; export function GetMediaFileByToken( fileToken: string, @@ -164,30 +39,6 @@ export function GetMediaFileByToken( }); } -export function ListVoiceConversionModels(): Promise< - Array | undefined -> { - return fetch(listV2V, { - method: "GET", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - }, - credentials: "include", - }) - .then((res) => res.json()) - .then((res) => { - const response: VoiceConversionModelListResponse = res; - if (!response.success) { - return; - } - return response?.models; - }) - .catch(() => { - return undefined; - }); -} - export function GenerateVoiceConversion( request: EnqueueVoiceConversionRequest, ): Promise { @@ -198,6 +49,7 @@ export function GenerateVoiceConversion( "Content-Type": "application/json", }, body: JSON.stringify(request), + credentials: "include", }) .then((res) => res.json()) .then((res) => { diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/voiceModelElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/voiceModelElement.tsx index 2c23f2029b..dd2c0347bb 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/voiceModelElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/itemTabs/AudioTab/voiceModelElement.tsx @@ -1,24 +1,15 @@ -import { TtsModelListItem } from "~/pages/PageEnigma/models/tts"; +import { Weight } from "~/models"; import { H4, H6 } from "~/components"; -import { VoiceConversionModelListItem } from "~/pages/PageEnigma/models"; export const VoiceModelElement = ({ model, onSelect, }: { - model: TtsModelListItem | VoiceConversionModelListItem; - onSelect: (item: TtsModelListItem | VoiceConversionModelListItem) => void; + model: Weight; + onSelect: (item: Weight) => void; }) => { - let creatorName: string | undefined; - if ("creator_display_name" in model) { - //case of TTS Models - creatorName = model.creator_display_name; - } else if ("creator" in model) { - //case of V2V Models - creatorName = model.creator.display_name; - } else { - creatorName = undefined; - } + const creatorName = model.creator.display_name; + return (