Audio Sidepanel v2.4 - Audio Items now sized appropriately in timeline (#264)

guaranteed audioItems to have length either from the server, or retrieve
it from the loading of the file,

the unit of mediaItem's length is FPS, adjusted the math for supplying
length in frames

Co-authored-by: wil <wil@storyteller.ai>

app/components/UploadComponent/UploadComponent.tsx

@@ -112,7 +112,7 @@ function UploadComponent({
         <>
           <div className="flex items-center gap-3 rounded-lg bg-brand-secondary p-3">
             <div className="grow">
-              <WaveformPlayer audio={audioUrl} />
+              <WaveformPlayer audio={audioUrl} hasPlayButton/>
             </div>
             <Tooltip content="Remove" position="top">
               <ButtonIcon

app/components/WaveformPlayer.tsx

@@ -5,10 +5,23 @@ import { ButtonIcon } from "~/components";
 import { environmentVariables } from "~/store";
 import { useSignals } from "@preact/signals-react/runtime";
 
-export const WaveformPlayer = ({ audio }: { audio: string }) => {
+function toTimeString(timecode: number):number {
+  return Math.round(timecode * 100) / 100;
+}
+
+export const WaveformPlayer = ({
+  hasPlayButton,
+  onLoad,
+  audio
+}: {
+  hasPlayButton?: boolean;
+  onLoad?:({duration}:{duration:number})=>void;
+  audio: string;
+}) => {
   useSignals();
   const waveSurferRef = useRef<WaveSurfer | undefined>(undefined);
-  const [isPlaying, toggleIsPlaying] = useState(false);
+  const [isPlaying, toggleIsPlaying] = useState<boolean>(false);
+  const [currenttime, setCurrenttime] = useState<number>(0);
 
   const containerRef = useCallback((node: HTMLDivElement) => {
     if (node) {
@@ -28,6 +41,7 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => {
       waveSurfer.load(newUrl);
       waveSurfer.on("ready", () => {
         waveSurferRef.current = waveSurfer;
+        if(onLoad) onLoad({duration: waveSurfer.getDuration()});
       });
       waveSurfer.on("play", () => {
         toggleIsPlaying(true);
@@ -35,9 +49,6 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => {
       waveSurfer.on("pause", () => {
         toggleIsPlaying(false);
       });
-      waveSurfer.on("finish", () => {
-        waveSurfer.seekTo(0);
-      });
     }
   }, []);
 
@@ -48,16 +59,20 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => {
     };
   }, []);
 
+  const duration = toTimeString(waveSurferRef.current ? waveSurferRef.current.getDuration() : 0);
+
   return (
     <div className="flex items-center gap-2 py-1">
-      <ButtonIcon
-        icon={isPlaying ? faCirclePause : faCirclePlay}
-        className="w-auto bg-transparent p-0 text-2xl hover:bg-transparent hover:opacity-75"
-        onClick={() => {
-          waveSurferRef.current?.playPause();
-        }}
-      />
-      <div ref={containerRef} className="w-full" />
+      {hasPlayButton &&
+        <ButtonIcon
+          icon={isPlaying ? faCirclePause : faCirclePlay}
+          className="w-auto bg-transparent p-0 text-2xl hover:bg-transparent hover:opacity-75"
+          onClick={() => {
+            waveSurferRef.current?.playPause();
+          }}
+        />
+      }
+      <div ref={containerRef} className="grow" />
     </div>
   );
 };

app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx

@@ -21,10 +21,10 @@ import {
 
 import { PageLibrary } from "./pageLibrary";
 import { PageAudioGeneration } from "./pageAudioGeneration";
-import { AudioTabPages, AudioPanelState, TtsState, V2VState } from "./types";
+import { AudioTabPages, AudioPanelState } from "./types";
 import { initialTtsState, initialV2VState } from "./values";
 import { TtsModelListItem } from "~/pages/PageEnigma/models/tts";
-import { VoiceConversionModelListItem } from "./typesImported";
+import { VoiceConversionModelListItem, MediaFile } from "./typesImported";
 import { PageSelectTtsModel } from "./pageSelectTtsModel";
 import { PageSelectV2VModel } from "./pageSelectV2VModel";
 
@@ -69,7 +69,12 @@ export const AudioTab = () => {
       }))
       return findNewItem !== undefined;
     }
-    ListAudioByUser(username, sessionToken).then((res:any[])=>{
+    function getLength(item:any){
+      return item.maybe_duration_millis 
+        ? item.maybe_duration_millis / 1000 * 60
+        : undefined;
+    }
+    ListAudioByUser(username, sessionToken).then((res:MediaFile[])=>{
       audioItemsFromServer.value = res.map(item=>{
         const morphedItem:AudioMediaItem = {
           version: 1,
@@ -78,9 +83,9 @@ export const AudioTab = () => {
           media_id: item.token,
           object_uuid: item.token,
           name: getTitle(item),
-          description: item.maybe_text_transcript,
+          description: item.maybe_text_transcript || "",
           publicBucketPath: item.public_bucket_path,
-          length: 25,
+          length: getLength(item),
           thumbnail: "/resources/placeholders/audio_placeholder.png",
           isMine: true,
           isNew: checkIsNew(item.token),

app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx

@@ -13,7 +13,11 @@ import {
 
 import { H5, H6 } from "~/components";
 import  { AudioTypePill } from "./audioTypePills"
-import { cancelNewFromAudioItem } from "~/pages/PageEnigma/store/mediaFromServer";
+import {
+  cancelNewFromAudioItem,
+  updateAudioItemLength,
+  updateDemoAudioItemLength,
+} from "~/pages/PageEnigma/store";
 
 function getGcsUrl(bucketRelativePath: string | undefined | null): string {
   let bucket = "vocodes-public";
@@ -63,7 +67,6 @@ export const AudioItemElement = ({ item }: Props) => {
   const onPointerDown = useCallback(
     (event: React.PointerEvent<HTMLDivElement>) => {
       if(item.isNew) {
-        console.log('pointerdown triggered');
         cancelNewFromAudioItem(item.media_id);
       }
       if (event.button === 0) {
@@ -89,11 +92,27 @@ export const AudioItemElement = ({ item }: Props) => {
       <div className="flex w-full flex-col gap-0.5 rounded-lg bg-assets-background p-2.5">
         <div className="flex justify-between">
           <AudioTypePill category={item.category} />
+          {/* <p>{item.length}</p> */}
           {item.isNew && <H6 className="text-media-is-new">New*</H6>}
         </div>
 
         {item.publicBucketPath && (
-          <WaveformPlayer audio={getGcsUrl(item.publicBucketPath)} />
+          <WaveformPlayer
+            hasPlayButton
+            audio={getGcsUrl(item.publicBucketPath)}
+            onLoad={item.length ?
+              undefined :
+              ({duration})=>{
+                // only do this for items that doesn't have a length
+                if(item.category === "demo"){
+                  updateDemoAudioItemLength(item.media_id, duration * 60);
+                }
+                else{
+                  updateAudioItemLength(item.media_id, duration * 60);
+                }
+              }
+          }
+          />
         )}
 
         <H5 className="text-overflow-ellipsis">{item.name}</H5>

app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts

@@ -130,6 +130,8 @@ export interface MediaFile {
   maybe_original_filename: string | null;
   maybe_creator_user: UserDetailsLight | null;
   maybe_prompt_token: string | null;
+  maybe_text_transcript: string | null;
+  maybe_duration_millis: number | null;
   cover_image: {
     maybe_cover_image_public_bucket_path: string | null;
     default_cover: {

app/pages/PageEnigma/models/assets.ts

@@ -7,7 +7,7 @@ export interface MediaItem {
   name: string;
   description?: string;
   publicBucketPath?: string;
-  length?: number;
+  length?: number; // unit of length is frames, as in FPS
   thumbnail?: string;
   imageIndex?: number;
   isMine?: boolean;

app/pages/PageEnigma/store/index.ts

@@ -4,6 +4,7 @@ export * from "./characterGroups";
 export * from "./dragAndDrop";
 export * from "./hotkeys";
 export * from "./media";
+export * from  "./mediaFromServer";
 export * from "./myMovies";
 export * from "./objectGroup";
 export * from "./selectedItem";

app/pages/PageEnigma/store/media.ts

@@ -2,6 +2,13 @@ import { signal } from "@preact/signals-core";
 import { AssetType, MediaItem, AudioMediaItem } from "~/pages/PageEnigma/models";
 import * as uuid from "uuid";
 
+export const updateDemoAudioItemLength = (mediaId:string, duration: number)=>{
+  const newList = audioItems.value.map((item)=>{
+    if( item.media_id === mediaId ) item.length = duration;
+    return item;
+  })
+  audioItems.value = [...newList];
+}
 export const animationItems = signal<MediaItem[]>([
   {
     version: 1,
@@ -452,7 +459,6 @@ export const audioItems = signal<AudioMediaItem[]>([
     version: 1,
     media_id: "m_403phjvjkbbaxxbz8y7r6qjay07mfd",
     type: AssetType.AUDIO,
-    length: 25,
     name: "Talk (Demo Sounds)",
     thumbnail: "/resources/placeholders/audio_placeholder.png",
     category: "demo",
@@ -463,7 +469,6 @@ export const audioItems = signal<AudioMediaItem[]>([
     version: 1,
     media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt",
     type: AssetType.AUDIO,
-    length: 25,
     name: "NCS Song",
     category: "demo",
     thumbnail: "/resources/placeholders/audio_placeholder.png",

app/pages/PageEnigma/store/mediaFromServer.ts

@@ -3,11 +3,17 @@ import { AudioMediaItem } from "~/pages/PageEnigma/models";
 export const audioItemsFromServer = signal<AudioMediaItem[]>([]);
 
 export const cancelNewFromAudioItem = (mediaId:string)=>{
-  console.log('function called');
   const newList = audioItemsFromServer.value.map((item)=>{
     if( item.media_id === mediaId ) item.isNew = false;
     return item;
   })
-  console.log(newList);
+  audioItemsFromServer.value = [...newList];
+}
+
+export const updateAudioItemLength = (mediaId:string, duration: number)=>{
+  const newList = audioItemsFromServer.value.map((item)=>{
+    if( item.media_id === mediaId ) item.length = duration;
+    return item;
+  })
   audioItemsFromServer.value = [...newList];
 }