fetching user media works, generate tts works

app/api.tsx

@@ -8,4 +8,7 @@ export const logout = `${apiHost}/v1/logout`;
 
 // TTS
 export const inferTts = `${apiHost}/tts/inference`;
-export const listTts = `${apiHost}/tts/list`;
+export const listTts = `${apiHost}/tts/list`;
+
+// User Media
+export const listMediaByUser = (username:string) => `${apiHost}/v1/media_files/list/user/${username}`;

app/contexts/Authentication/Authentication.tsx

@@ -31,8 +31,8 @@ export const AuthenticationContext = createContext<{
 });
 
 export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
-  console.log('auth provider rerender');
-  const [authCookies, setAuthCookie, removeAuthCookie] = useCookies([STORAGE_KEYS.USER_INFO]);
+  // console.log('auth provider rerender');
+  const [, setAuthCookie, removeAuthCookie] = useCookies([STORAGE_KEYS.USER_INFO]);
   const [authState, setAuthState] = useState<AuthState>({});
 
   const loginAndGetUserInfo = useCallback((
@@ -78,9 +78,9 @@ export const AuthenticationProvider = ({children}:{children:ReactNode})=>{
     if(authState.isLoggedIn === undefined){
       //Persist Login if Session Data exist
       const sessionData = localStorage.getItem(STORAGE_KEYS.USER_INFO);
-      console.log(`Session Data: ${sessionData?.substring(0, 30)}${sessionData ? '...':''}`);
+      // console.log(`Session Data: ${sessionData?.substring(0, 30)}${sessionData ? '...':''}`);
       if (sessionData !== null){
-        console.log('Getting session to persist login');
+        // console.log('Getting session to persist login');
         GetSession().then((
           res: SessionResponse
         )=>{

app/pages/PageEnigma/comps/SidePanelTabs/AudioElements.tsx

@@ -1,30 +0,0 @@
-import { useContext, useEffect, useState } from "react";
-import { AnimationElement } from "./AnimationElement";
-import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
-import { useSignals } from "@preact/signals-react/runtime";
-import { timelineHeight } from "~/pages/PageEnigma/store";
-import { ClipType } from "~/pages/PageEnigma/models/track";
-
-export const AudioElements = () => {
-  const { audioClips } = useContext(TrackContext);
-  useSignals();
-  const [height, setHeight] = useState(0);
-
-  useEffect(() => {
-    setHeight(window.outerHeight - timelineHeight.value);
-  }, []);
-
-  return (
-    <div className="flex flex-wrap gap-3 overflow-y-auto" style={{ height }}>
-      {audioClips.map((clip) => {
-        return (
-          <AnimationElement
-            key={clip.media_id}
-            clip={clip}
-            type={ClipType.AUDIO}
-          />
-        );
-      })}
-    </div>
-  );
-};

app/pages/PageEnigma/comps/SidePanelTabs/TabAudio.tsx

@@ -1,68 +0,0 @@
-import { useContext, useEffect, useState } from "react";
-import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
-import { APPUI_ACTION_TYPES } from "../../reducers";
-import { AppUiContext } from "../../contexts/AppUiContext";
-import { useSignals } from "@preact/signals-react/runtime";
-import { timelineHeight } from "~/pages/PageEnigma/store";
-
-import { AnimationElement } from "./AnimationElement";
-import { Button, Label, P } from "~/components";
-
-export const TabAudio = () => {
-  const { audioClips } = useContext(TrackContext);
-  const [appUiState, dispatchAppUiState] = useContext(AppUiContext);
-
-  useSignals();
-  const [height, setHeight] = useState(0);
-
-  useEffect(() => {
-    setHeight(window.outerHeight - timelineHeight.value);
-  }, []);
-
-  const openTTSPanel = () =>{
-    dispatchAppUiState({
-      type: APPUI_ACTION_TYPES.OPEN_DIALOGUE_TTS
-    })
-  };
-
-  return (
-    <div className="flex flex-col gap-3">
-      <div className="flex flex-col">
-        <Label>Generate</Label>
-        <Button
-          onClick={openTTSPanel}
-          variant="secondary"
-        >
-          Create TTS Audio
-        </Button>
-        <span className="w-full h-3" />
-        <Button
-          disabled
-          variant="secondary"
-        >
-          Create Voice to Voice Audio
-        </Button>
-      </div>
-      <div className="flex flex-col mt-1">
-        <Label>Preset Dialogues</Label>
-      
-        <div className="flex flex-wrap gap-2">
-          {audioClips.map((clip) => {
-            return (
-              <AnimationElement key={clip.media_id} clip={clip} type="audio" />
-            );
-        })}
-        </div>
-      </div>
-
-      <div className="flex flex-col mt-1">
-        <Label>My Dialogues</Label>
-        <div className="flex justify-center items-center text-center w-full h-40">
-          <P className="text-brand-secondary-300"> No audio generated yet</P>
-        </div>
-      </div>
-
-      <Button className="w-fit m-auto px-6 py-2"> Add to Lip Sync Track</Button>
-    </div>
-  );
-};

app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/TabAudio.tsx

@@ -0,0 +1,101 @@
+import { useContext, useEffect, useState } from "react";
+import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext";
+
+import { APPUI_ACTION_TYPES } from "../../../reducers";
+import { AppUiContext } from "../../../contexts/AppUiContext";
+import { AuthenticationContext } from "~/contexts/Authentication";
+import { useSignals } from "@preact/signals-react/runtime";
+import { timelineHeight } from "~/pages/PageEnigma/store";
+
+import { Button, Label, P } from "~/components";
+import { ClipType } from "../../../models/track";
+import { AudioElement } from "./AudioElement";
+import { ListAudioByUser, MediaFile } from "./listAudioByUser";
+
+export const TabAudio = () => {
+  const { audioClips } = useContext(TrackContext);
+  const { authState } = useContext(AuthenticationContext);
+  const [, dispatchAppUiState] = useContext(AppUiContext);
+  const [ userAudioClips, setUserAudioClips] = useState<MediaFile[]>([]);
+
+  useSignals();
+  const [height, setHeight] = useState(0);
+
+  useEffect(() => {
+    setHeight(window.outerHeight - timelineHeight.value);
+    if(authState.userInfo && authState.userInfo.username)
+      ListAudioByUser(authState.userInfo.username)
+        .then(res=>{
+          setUserAudioClips([...res]);
+        });
+  }, []);
+
+  const openTTSPanel = () =>{
+    dispatchAppUiState({
+      type: APPUI_ACTION_TYPES.OPEN_DIALOGUE_TTS
+    })
+  };
+
+  return (
+    <div 
+      className="flex flex-col gap-3"
+    >
+      <div className="flex flex-col">
+        <Label>Generate</Label>
+        <Button
+          onClick={openTTSPanel}
+          variant="secondary"
+        >
+          Create TTS Audio
+        </Button>
+        <span className="w-full h-3" />
+        <Button
+          disabled
+          variant="secondary"
+        >
+          Create Voice to Voice Audio
+        </Button>
+      </div>
+      <div className="flex flex-col mt-1">
+        <Label>Preset Dialogues</Label>
+      
+        <div className="flex flex-wrap gap-2">
+          {audioClips.map((clip) => {
+            return (
+              <AudioElement
+                key={clip.media_id}
+                clip={clip}
+                type={ClipType.AUDIO}
+              />
+            );
+        })}
+        </div>
+      </div>
+
+      <div className="flex flex-col mt-1">
+        <Label>My Dialogues</Label>
+        {userAudioClips.length === 0 && 
+          <div className="flex justify-center items-center text-center w-full h-40">
+            <P className="text-brand-secondary-300"> No audio generated yet</P>
+          </div>
+        }
+        { userAudioClips.length >= 0 &&
+          <div>
+            {userAudioClips.map(item =>
+              <>
+                <P>{item.origin['maybe_model'].title || "No Name"}</P>
+                <AudioElement 
+                  key={item.token}
+                  clip={item}
+                  type={ClipType.AUDIO}
+                />
+              </>
+            )}
+          </div>
+        }
+      </div>
+
+      <Button className="w-fit m-auto px-6 py-2"> Add to Lip Sync Track</Button>
+    </div>
+  );
+};

app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/index.tsx

@@ -0,0 +1 @@
+export { TabAudio } from "./TabAudio";

app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/listAudioByUser.tsx

@@ -0,0 +1,55 @@
+import { listMediaByUser } from "~/api";
+import {
+  MediaFileType,
+  MediaFileClass,
+  MediaFileSubtype,
+  UserDetailsLight,
+  WeightCategory,
+  WeightType
+} from "./types";
+
+
+export interface MediaFile {
+  token: string;
+  media_type: MediaFileType;
+  media_class: MediaFileClass | null;
+  maybe_media_subtype: MediaFileSubtype | null;
+  public_bucket_path: string;
+  maybe_engine_extension: string | null;
+  maybe_batch_token: string;
+  maybe_title: string | null;
+  maybe_original_filename: string | null;
+  maybe_creator_user: UserDetailsLight | null;
+  maybe_prompt_token: string | null;
+  creator_set_visibility: string;
+  created_at: Date;
+  updated_at: Date;
+  maybe_model_weight_info: {
+    title: string;
+    weight_token: string;
+    weight_category: WeightCategory;
+    weight_type: WeightType;
+    maybe_weight_creator: UserDetailsLight;
+    maybe_cover_image_public_bucket_path: string;
+  };
+}
+
+export const ListAudioByUser = async(username:string) => {
+  return await fetch(listMediaByUser(username),{
+    method: 'GET',
+    headers: {
+      "Accept": "application/json",
+    },
+    credentials: 'include'
+  })
+  .then(res => res.json())
+  .then(res => { 
+    if(res.success && res.results){
+      return res.results.filter((item:MediaFile)=>item['media_type']==='audio');
+    }else{
+      Promise.reject();
+    }
+  })
+  .catch(e => ({ success : false }));
+}
+

app/pages/PageEnigma/comps/SidePanelTabs/TabAudio/types.tsx

@@ -0,0 +1,124 @@
+export enum WeightCategory {
+  TTS = "text_to_speech",
+  VOCODER = "vocoder",
+  VC = "voice_conversion",
+  ZS = "zero_shot",
+  SD = "image_generation",
+  WF = "workflow_config",
+}
+
+
+export enum MediaFileClass {
+  /// Unknown (default value)
+ /// This will be present until we migrate all old files.
+ Unknown = "unknown",
+
+ /// Audio files: wav, mp3, etc.
+ Audio = "audio",
+
+ /// Image files: png, jpeg, etc.
+ Image = "image",
+
+ /// Video files: mp4, etc.
+ Video = "video",
+
+ /// Engine "animations"
+ Animation = "animation",
+
+ /// Engine "characters"
+ Character = "character",
+
+ /// Engine "prop" items
+ Prop = "prop",
+
+ /// Engine scenes (internal and external scenes)
+ Scene = "scene",
+}
+
+
+export enum MediaFileType {
+  Audio = "audio",
+  Video = "video",
+  Image = "image",
+
+  // BVH is a very popular file format for motion capture.
+  // It is compatible with our engine.
+  BVH = "bvh",
+
+  // GLTF is the text-based format of GLB. Unforunately it requires a
+  // second file to include the visual data, so we strongly discourage
+  // the use of this format in favor of GLB.
+  GLTF = "gltf",
+
+  // GLB is the format we convert FBX files into. It's a very popular
+  // open source 3D file format.
+  GLB = "glb",
+
+  // FBX isn't supported by Storyteller Engine, but we can get
+  // uploads in this format. It's a very popular 3D graphics format.
+  FBX = "fbx",
+
+  // Full Storyteller Engine scenes (RON = Rusty Object Notation)
+  // This is a temporary format that will eventually go away.
+  SceneRon = "scene_ron",
+
+  None = "none",
+}
+
+
+export enum MediaFileSubtype {
+  /// Animation file from Mixamo
+  /// Primarily used for FBX and GLB.
+  Mixamo = "mixamo",
+
+  /// Animation file from MocapNet
+  /// Primarily used for BVH.
+  MocapNet = "mocap_net",
+
+  /// Generic animation case
+  /// Used for BVH files, but can also pertain to animation-only files of other types.
+  AnimationOnly = "animation_only",
+
+  // TODO(bt,2024-03-08): Migrate records and code, then remove
+  /// DEPRECATED: Use `SceneImport` instead.
+  Scene = "scene",
+
+  /// Generic 3D scene file.
+  /// Can pertain to glTF, glB, FBX, etc.
+  SceneImport = "scene_import",
+
+  /// Native Storyteller scene format.
+  /// Typically stored in a `.scn.ron` file.
+  StorytellerScene = "storyteller_scene",
+}
+
+/// A common type returned by several endpoints.
+/// Basic information to display a user and their avatar.
+export interface UserDetailsLight {
+  user_token: string,
+  /// Username (lowercase)
+  username: string,
+  /// Username with user-specified capitalization
+  display_name: string,
+  gravatar_hash: string,
+  default_avatar: DefaultAvatarInfo,
+
+}
+
+export interface DefaultAvatarInfo {
+  image_index: number,
+  color_index: number,
+}
+
+
+export enum WeightType {
+  TT2 = "tt2",
+  HIFIGAN_TT2 = "hifigan_tt2",
+  RVCv2 = "rvc_v2",
+  SD_15 = "sd_1.5",
+  SDXL = "sdxl",
+  SVC = "so_vits_svc",
+  LORA = "loRA",
+  VALL_E = "vall_e",
+  NONE = "none",
+}

app/pages/PageEnigma/comps/SidePanelTabs/index.tsx

@@ -1,7 +1,7 @@
 import { Tabs } from "~/modules/Tabs";
 import { TabStylization } from "./TabStylization";
 import { AnimationElements } from "./AnimationElements";
-import { TabAudio } from "~/pages/PageEnigma/comps/SidePanelTabs/TabAudio";
+import { TabAudio } from "./TabAudio";
 
 export const SidePanelTabs = () => {
   return (