From 24cb2906c6ffe5fa048fbc933d1ae98d33a38287 Mon Sep 17 00:00:00 2001 From: wil Date: Mon, 29 Apr 2024 13:43:23 -0400 Subject: [PATCH] Audio Sidepanel v2.4 - Audio Items now sized appropriately in timeline (#264) guaranteed audioItems to have length either from the server, or retrieve it from the loading of the file, the unit of mediaItem's length is FPS, adjusted the math for supplying length in frames Co-authored-by: wil --- .../UploadComponent/UploadComponent.tsx | 2 +- app/components/WaveformPlayer.tsx | 41 +++++++++++++------ .../comps/SidePanelTabs/AudioTab/AudioTab.tsx | 15 ++++--- .../AudioTab/audioItemElement.tsx | 25 +++++++++-- .../SidePanelTabs/AudioTab/typesImported.ts | 2 + app/pages/PageEnigma/models/assets.ts | 2 +- app/pages/PageEnigma/store/index.ts | 1 + app/pages/PageEnigma/store/media.ts | 9 +++- app/pages/PageEnigma/store/mediaFromServer.ts | 10 ++++- 9 files changed, 80 insertions(+), 27 deletions(-) diff --git a/app/components/UploadComponent/UploadComponent.tsx b/app/components/UploadComponent/UploadComponent.tsx index d84782de4c..b54c866070 100644 --- a/app/components/UploadComponent/UploadComponent.tsx +++ b/app/components/UploadComponent/UploadComponent.tsx @@ -112,7 +112,7 @@ function UploadComponent({ <>
- +
{ +function toTimeString(timecode: number):number { + return Math.round(timecode * 100) / 100; +} + +export const WaveformPlayer = ({ + hasPlayButton, + onLoad, + audio +}: { + hasPlayButton?: boolean; + onLoad?:({duration}:{duration:number})=>void; + audio: string; +}) => { useSignals(); const waveSurferRef = useRef(undefined); - const [isPlaying, toggleIsPlaying] = useState(false); + const [isPlaying, toggleIsPlaying] = useState(false); + const [currenttime, setCurrenttime] = useState(0); const containerRef = useCallback((node: HTMLDivElement) => { if (node) { @@ -28,6 +41,7 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => { waveSurfer.load(newUrl); waveSurfer.on("ready", () => { waveSurferRef.current = waveSurfer; + if(onLoad) onLoad({duration: waveSurfer.getDuration()}); }); waveSurfer.on("play", () => { toggleIsPlaying(true); @@ -35,9 +49,6 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => { waveSurfer.on("pause", () => { toggleIsPlaying(false); }); - waveSurfer.on("finish", () => { - waveSurfer.seekTo(0); - }); } }, []); @@ -48,16 +59,20 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => { }; }, []); + const duration = toTimeString(waveSurferRef.current ? waveSurferRef.current.getDuration() : 0); + return (
- { - waveSurferRef.current?.playPause(); - }} - /> -
+ {hasPlayButton && + { + waveSurferRef.current?.playPause(); + }} + /> + } +
); }; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx index 4055a56804..8f86fa93ea 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx @@ -21,10 +21,10 @@ import { import { PageLibrary } from "./pageLibrary"; import { PageAudioGeneration } from "./pageAudioGeneration"; -import { AudioTabPages, AudioPanelState, TtsState, V2VState } from "./types"; +import { AudioTabPages, AudioPanelState } from "./types"; import { initialTtsState, initialV2VState } from "./values"; import { TtsModelListItem } from "~/pages/PageEnigma/models/tts"; -import { VoiceConversionModelListItem } from "./typesImported"; +import { VoiceConversionModelListItem, MediaFile } from "./typesImported"; import { PageSelectTtsModel } from "./pageSelectTtsModel"; import { PageSelectV2VModel } from "./pageSelectV2VModel"; @@ -69,7 +69,12 @@ export const AudioTab = () => { })) return findNewItem !== undefined; } - ListAudioByUser(username, sessionToken).then((res:any[])=>{ + function getLength(item:any){ + return item.maybe_duration_millis + ? item.maybe_duration_millis / 1000 * 60 + : undefined; + } + ListAudioByUser(username, sessionToken).then((res:MediaFile[])=>{ audioItemsFromServer.value = res.map(item=>{ const morphedItem:AudioMediaItem = { version: 1, @@ -78,9 +83,9 @@ export const AudioTab = () => { media_id: item.token, object_uuid: item.token, name: getTitle(item), - description: item.maybe_text_transcript, + description: item.maybe_text_transcript || "", publicBucketPath: item.public_bucket_path, - length: 25, + length: getLength(item), thumbnail: "/resources/placeholders/audio_placeholder.png", isMine: true, isNew: checkIsNew(item.token), diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx index 1c86d4f498..2c98881a41 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx @@ -13,7 +13,11 @@ import { import { H5, H6 } from "~/components"; import { AudioTypePill } from "./audioTypePills" -import { cancelNewFromAudioItem } from "~/pages/PageEnigma/store/mediaFromServer"; +import { + cancelNewFromAudioItem, + updateAudioItemLength, + updateDemoAudioItemLength, +} from "~/pages/PageEnigma/store"; function getGcsUrl(bucketRelativePath: string | undefined | null): string { let bucket = "vocodes-public"; @@ -63,7 +67,6 @@ export const AudioItemElement = ({ item }: Props) => { const onPointerDown = useCallback( (event: React.PointerEvent) => { if(item.isNew) { - console.log('pointerdown triggered'); cancelNewFromAudioItem(item.media_id); } if (event.button === 0) { @@ -89,11 +92,27 @@ export const AudioItemElement = ({ item }: Props) => {
+ {/*

{item.length}

*/} {item.isNew &&
New*
}
{item.publicBucketPath && ( - + { + // only do this for items that doesn't have a length + if(item.category === "demo"){ + updateDemoAudioItemLength(item.media_id, duration * 60); + } + else{ + updateAudioItemLength(item.media_id, duration * 60); + } + } + } + /> )}
{item.name}
diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts index d853e75275..15a60fb50f 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts @@ -130,6 +130,8 @@ export interface MediaFile { maybe_original_filename: string | null; maybe_creator_user: UserDetailsLight | null; maybe_prompt_token: string | null; + maybe_text_transcript: string | null; + maybe_duration_millis: number | null; cover_image: { maybe_cover_image_public_bucket_path: string | null; default_cover: { diff --git a/app/pages/PageEnigma/models/assets.ts b/app/pages/PageEnigma/models/assets.ts index 376549aa1b..bb81da02a1 100644 --- a/app/pages/PageEnigma/models/assets.ts +++ b/app/pages/PageEnigma/models/assets.ts @@ -7,7 +7,7 @@ export interface MediaItem { name: string; description?: string; publicBucketPath?: string; - length?: number; + length?: number; // unit of length is frames, as in FPS thumbnail?: string; imageIndex?: number; isMine?: boolean; diff --git a/app/pages/PageEnigma/store/index.ts b/app/pages/PageEnigma/store/index.ts index 07628a1914..fb152d4bfe 100644 --- a/app/pages/PageEnigma/store/index.ts +++ b/app/pages/PageEnigma/store/index.ts @@ -4,6 +4,7 @@ export * from "./characterGroups"; export * from "./dragAndDrop"; export * from "./hotkeys"; export * from "./media"; +export * from "./mediaFromServer"; export * from "./myMovies"; export * from "./objectGroup"; export * from "./selectedItem"; diff --git a/app/pages/PageEnigma/store/media.ts b/app/pages/PageEnigma/store/media.ts index 8ed2608793..ea557810fb 100644 --- a/app/pages/PageEnigma/store/media.ts +++ b/app/pages/PageEnigma/store/media.ts @@ -2,6 +2,13 @@ import { signal } from "@preact/signals-core"; import { AssetType, MediaItem, AudioMediaItem } from "~/pages/PageEnigma/models"; import * as uuid from "uuid"; +export const updateDemoAudioItemLength = (mediaId:string, duration: number)=>{ + const newList = audioItems.value.map((item)=>{ + if( item.media_id === mediaId ) item.length = duration; + return item; + }) + audioItems.value = [...newList]; +} export const animationItems = signal([ { version: 1, @@ -452,7 +459,6 @@ export const audioItems = signal([ version: 1, media_id: "m_403phjvjkbbaxxbz8y7r6qjay07mfd", type: AssetType.AUDIO, - length: 25, name: "Talk (Demo Sounds)", thumbnail: "/resources/placeholders/audio_placeholder.png", category: "demo", @@ -463,7 +469,6 @@ export const audioItems = signal([ version: 1, media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt", type: AssetType.AUDIO, - length: 25, name: "NCS Song", category: "demo", thumbnail: "/resources/placeholders/audio_placeholder.png", diff --git a/app/pages/PageEnigma/store/mediaFromServer.ts b/app/pages/PageEnigma/store/mediaFromServer.ts index 8974da4e57..661ec27f14 100644 --- a/app/pages/PageEnigma/store/mediaFromServer.ts +++ b/app/pages/PageEnigma/store/mediaFromServer.ts @@ -3,11 +3,17 @@ import { AudioMediaItem } from "~/pages/PageEnigma/models"; export const audioItemsFromServer = signal([]); export const cancelNewFromAudioItem = (mediaId:string)=>{ - console.log('function called'); const newList = audioItemsFromServer.value.map((item)=>{ if( item.media_id === mediaId ) item.isNew = false; return item; }) - console.log(newList); + audioItemsFromServer.value = [...newList]; +} + +export const updateAudioItemLength = (mediaId:string, duration: number)=>{ + const newList = audioItemsFromServer.value.map((item)=>{ + if( item.media_id === mediaId ) item.length = duration; + return item; + }) audioItemsFromServer.value = [...newList]; } \ No newline at end of file