From d46a5946bbabe8485605602cf2724c2a6bbbb388 Mon Sep 17 00:00:00 2001 From: wil Date: Thu, 4 Apr 2024 22:55:53 -0400 Subject: [PATCH] fetching user media works, generate tts works --- app/api.tsx | 5 +- .../Authentication/Authentication.tsx | 8 +- .../comps/SidePanelTabs/AudioElements.tsx | 30 ----- .../comps/SidePanelTabs/TabAudio.tsx | 68 ---------- .../{ => TabAudio}/AudioElement.tsx | 0 .../comps/SidePanelTabs/TabAudio/TabAudio.tsx | 101 ++++++++++++++ .../comps/SidePanelTabs/TabAudio/index.tsx | 1 + .../TabAudio/listAudioByUser.tsx | 55 ++++++++ .../comps/SidePanelTabs/TabAudio/types.tsx | 124 ++++++++++++++++++ .../PageEnigma/comps/SidePanelTabs/index.tsx | 2 +- 10 files changed, 290 insertions(+), 104 deletions(-) delete mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx delete mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/TabAudio.tsx rename app/pages/PageEnigma/comps/SidePanelTabs/{ => TabAudio}/AudioElement.tsx (100%) create mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/TabAudio.tsx create mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/index.tsx create mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/listAudioByUser.tsx create mode 100644 app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/types.tsx diff --git a/app/api.tsx b/app/api.tsx index 666b25afbb..0e0bcc77a1 100644 --- a/app/api.tsx +++ b/app/api.tsx @@ -8,4 +8,7 @@ export const logout = `${apiHost}/v1/logout`; // TTS export const inferTts = `${apiHost}/tts/inference`; -export const listTts = `${apiHost}/tts/list`; \ No newline at end of file +export const listTts = `${apiHost}/tts/list`; + +// User Media +export const listMediaByUser = (username:string) => `${apiHost}/v1/media_files/list/user/${username}`; \ No newline at end of file diff --git a/app/contexts/Authentication/Authentication.tsx b/app/contexts/Authentication/Authentication.tsx index 8e770955d1..615a1bed5d 100644 --- a/app/contexts/Authentication/Authentication.tsx +++ b/app/contexts/Authentication/Authentication.tsx @@ -31,8 +31,8 @@ export const AuthenticationContext = createContext<{ }); export const AuthenticationProvider = ({children}:{children:ReactNode})=>{ - console.log('auth provider rerender'); - const [authCookies, setAuthCookie, removeAuthCookie] = useCookies([STORAGE_KEYS.USER_INFO]); + // console.log('auth provider rerender'); + const [, setAuthCookie, removeAuthCookie] = useCookies([STORAGE_KEYS.USER_INFO]); const [authState, setAuthState] = useState({}); const loginAndGetUserInfo = useCallback(( @@ -78,9 +78,9 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{ if(authState.isLoggedIn === undefined){ //Persist Login if Session Data exist const sessionData = localStorage.getItem(STORAGE_KEYS.USER_INFO); - console.log(`Session Data: ${sessionData?.substring(0, 30)}${sessionData ? '...':''}`); + // console.log(`Session Data: ${sessionData?.substring(0, 30)}${sessionData ? '...':''}`); if (sessionData !== null){ - console.log('Getting session to persist login'); + // console.log('Getting session to persist login'); GetSession().then(( res: SessionResponse )=>{ diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx deleted file mode 100644 index ee3eb38341..0000000000 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { useContext, useEffect, useState } from "react"; -import { AnimationElement } from "./AnimationElement"; -import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; -import { useSignals } from "@preact/signals-react/runtime"; -import { timelineHeight } from "~/pages/PageEnigma/store"; -import { ClipType } from "~/pages/PageEnigma/models/track"; - -export const AudioElements = () => { - const { audioClips } = useContext(TrackContext); - useSignals(); - const [height, setHeight] = useState(0); - - useEffect(() => { - setHeight(window.outerHeight - timelineHeight.value); - }, []); - - return ( -
- {audioClips.map((clip) => { - return ( - - ); - })} -
- ); -}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio.tsx deleted file mode 100644 index c466cb9c9e..0000000000 --- a/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio.tsx +++ /dev/null @@ -1,68 +0,0 @@ -import { useContext, useEffect, useState } from "react"; -import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; -import { APPUI_ACTION_TYPES } from "../../reducers"; -import { AppUiContext } from "../../contexts/AppUiContext"; -import { useSignals } from "@preact/signals-react/runtime"; -import { timelineHeight } from "~/pages/PageEnigma/store"; - -import { AnimationElement } from "./AnimationElement"; -import { Button, Label, P } from "~/components"; - -export const TabAudio = () => { - const { audioClips } = useContext(TrackContext); - const [appUiState, dispatchAppUiState] = useContext(AppUiContext); - - useSignals(); - const [height, setHeight] = useState(0); - - useEffect(() => { - setHeight(window.outerHeight - timelineHeight.value); - }, []); - - const openTTSPanel = () =>{ - dispatchAppUiState({ - type: APPUI_ACTION_TYPES.OPEN_DIALOGUE_TTS - }) - }; - - return ( -
-
- - - - -
-
- - -
- {audioClips.map((clip) => { - return ( - - ); - })} -
-
- -
- -
-

No audio generated yet

-
-
- - -
- ); -}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/AudioElement.tsx similarity index 100% rename from app/pages/PageEnigma/comps/SidePanelTabs/AudioElement.tsx rename to app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/AudioElement.tsx diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/TabAudio.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/TabAudio.tsx new file mode 100644 index 0000000000..6a564263d2 --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/TabAudio.tsx @@ -0,0 +1,101 @@ +import { useContext, useEffect, useState } from "react"; +import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; + +import { APPUI_ACTION_TYPES } from "../../../reducers"; +import { AppUiContext } from "../../../contexts/AppUiContext"; +import { AuthenticationContext } from "~/contexts/Authentication"; +import { useSignals } from "@preact/signals-react/runtime"; +import { timelineHeight } from "~/pages/PageEnigma/store"; + +import { Button, Label, P } from "~/components"; +import { ClipType } from "../../../models/track"; +import { AudioElement } from "./AudioElement"; +import { ListAudioByUser, MediaFile } from "./listAudioByUser"; + +export const TabAudio = () => { + const { audioClips } = useContext(TrackContext); + const { authState } = useContext(AuthenticationContext); + const [, dispatchAppUiState] = useContext(AppUiContext); + const [ userAudioClips, setUserAudioClips] = useState([]); + + useSignals(); + const [height, setHeight] = useState(0); + + useEffect(() => { + setHeight(window.outerHeight - timelineHeight.value); + if(authState.userInfo && authState.userInfo.username) + ListAudioByUser(authState.userInfo.username) + .then(res=>{ + setUserAudioClips([...res]); + }); + }, []); + + const openTTSPanel = () =>{ + dispatchAppUiState({ + type: APPUI_ACTION_TYPES.OPEN_DIALOGUE_TTS + }) + }; + + return ( +
+
+ + + + +
+
+ + +
+ {audioClips.map((clip) => { + return ( + + ); + })} +
+
+ +
+ + {userAudioClips.length === 0 && +
+

No audio generated yet

+
+ } + { userAudioClips.length >= 0 && +
+ {userAudioClips.map(item => + <> +

{item.origin['maybe_model'].title || "No Name"}

+ + + )} +
+ } +
+ + +
+ ); +}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/index.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/index.tsx new file mode 100644 index 0000000000..803333183c --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/index.tsx @@ -0,0 +1 @@ +export { TabAudio } from "./TabAudio"; \ No newline at end of file diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/listAudioByUser.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/listAudioByUser.tsx new file mode 100644 index 0000000000..b277c03941 --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/listAudioByUser.tsx @@ -0,0 +1,55 @@ +import { listMediaByUser } from "~/api"; +import { + MediaFileType, + MediaFileClass, + MediaFileSubtype, + UserDetailsLight, + WeightCategory, + WeightType +} from "./types"; + + +export interface MediaFile { + token: string; + media_type: MediaFileType; + media_class: MediaFileClass | null; + maybe_media_subtype: MediaFileSubtype | null; + public_bucket_path: string; + maybe_engine_extension: string | null; + maybe_batch_token: string; + maybe_title: string | null; + maybe_original_filename: string | null; + maybe_creator_user: UserDetailsLight | null; + maybe_prompt_token: string | null; + creator_set_visibility: string; + created_at: Date; + updated_at: Date; + maybe_model_weight_info: { + title: string; + weight_token: string; + weight_category: WeightCategory; + weight_type: WeightType; + maybe_weight_creator: UserDetailsLight; + maybe_cover_image_public_bucket_path: string; + }; +} + +export const ListAudioByUser = async(username:string) => { + return await fetch(listMediaByUser(username),{ + method: 'GET', + headers: { + "Accept": "application/json", + }, + credentials: 'include' + }) + .then(res => res.json()) + .then(res => { + if(res.success && res.results){ + return res.results.filter((item:MediaFile)=>item['media_type']==='audio'); + }else{ + Promise.reject(); + } + }) + .catch(e => ({ success : false })); +} + diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/types.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/types.tsx new file mode 100644 index 0000000000..098a7e0dfe --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/types.tsx @@ -0,0 +1,124 @@ +export enum WeightCategory { + TTS = "text_to_speech", + VOCODER = "vocoder", + VC = "voice_conversion", + ZS = "zero_shot", + SD = "image_generation", + WF = "workflow_config", +} + + +export enum MediaFileClass { + /// Unknown (default value) + /// This will be present until we migrate all old files. + Unknown = "unknown", + + /// Audio files: wav, mp3, etc. + Audio = "audio", + + /// Image files: png, jpeg, etc. + Image = "image", + + /// Video files: mp4, etc. + Video = "video", + + /// Engine "animations" + Animation = "animation", + + /// Engine "characters" + Character = "character", + + /// Engine "prop" items + Prop = "prop", + + /// Engine scenes (internal and external scenes) + Scene = "scene", +} + + +export enum MediaFileType { + Audio = "audio", + Video = "video", + Image = "image", + + // BVH is a very popular file format for motion capture. + // It is compatible with our engine. + BVH = "bvh", + + // GLTF is the text-based format of GLB. Unforunately it requires a + // second file to include the visual data, so we strongly discourage + // the use of this format in favor of GLB. + GLTF = "gltf", + + // GLB is the format we convert FBX files into. It's a very popular + // open source 3D file format. + GLB = "glb", + + // FBX isn't supported by Storyteller Engine, but we can get + // uploads in this format. It's a very popular 3D graphics format. + FBX = "fbx", + + // Full Storyteller Engine scenes (RON = Rusty Object Notation) + // This is a temporary format that will eventually go away. + SceneRon = "scene_ron", + + None = "none", +} + + +export enum MediaFileSubtype { + /// Animation file from Mixamo + /// Primarily used for FBX and GLB. + Mixamo = "mixamo", + + /// Animation file from MocapNet + /// Primarily used for BVH. + MocapNet = "mocap_net", + + /// Generic animation case + /// Used for BVH files, but can also pertain to animation-only files of other types. + AnimationOnly = "animation_only", + + // TODO(bt,2024-03-08): Migrate records and code, then remove + /// DEPRECATED: Use `SceneImport` instead. + Scene = "scene", + + /// Generic 3D scene file. + /// Can pertain to glTF, glB, FBX, etc. + SceneImport = "scene_import", + + /// Native Storyteller scene format. + /// Typically stored in a `.scn.ron` file. + StorytellerScene = "storyteller_scene", +} + +/// A common type returned by several endpoints. +/// Basic information to display a user and their avatar. +export interface UserDetailsLight { + user_token: string, + /// Username (lowercase) + username: string, + /// Username with user-specified capitalization + display_name: string, + gravatar_hash: string, + default_avatar: DefaultAvatarInfo, + +} + +export interface DefaultAvatarInfo { + image_index: number, + color_index: number, +} + + +export enum WeightType { + TT2 = "tt2", + HIFIGAN_TT2 = "hifigan_tt2", + RVCv2 = "rvc_v2", + SD_15 = "sd_1.5", + SDXL = "sdxl", + SVC = "so_vits_svc", + LORA = "loRA", + VALL_E = "vall_e", + NONE = "none", +} diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/index.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/index.tsx index 02911e9b53..6645d5a559 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/index.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/index.tsx @@ -1,7 +1,7 @@ import { Tabs } from "~/modules/Tabs"; import { TabStylization } from "./TabStylization"; import { AnimationElements } from "./AnimationElements"; -import { TabAudio } from "~/pages/PageEnigma/comps/SidePanelTabs/TabAudio"; +import { TabAudio } from "./TabAudio"; export const SidePanelTabs = () => { return (