From 24cb2906c6ffe5fa048fbc933d1ae98d33a38287 Mon Sep 17 00:00:00 2001 From: wil Date: Mon, 29 Apr 2024 13:43:23 -0400 Subject: [PATCH] Audio Sidepanel v2.4 - Audio Items now sized appropriately in timeline (#264) guaranteed audioItems to have length either from the server, or retrieve it from the loading of the file, the unit of mediaItem's length is FPS, adjusted the math for supplying length in frames Co-authored-by: wil --- .../UploadComponent/UploadComponent.tsx | 2 +- app/components/WaveformPlayer.tsx | 41 +++++++++++++------ .../comps/SidePanelTabs/AudioTab/AudioTab.tsx | 15 ++++--- .../AudioTab/audioItemElement.tsx | 25 +++++++++-- .../SidePanelTabs/AudioTab/typesImported.ts | 2 + app/pages/PageEnigma/models/assets.ts | 2 +- app/pages/PageEnigma/store/index.ts | 1 + app/pages/PageEnigma/store/media.ts | 9 +++- app/pages/PageEnigma/store/mediaFromServer.ts | 10 ++++- 9 files changed, 80 insertions(+), 27 deletions(-) diff --git a/app/components/UploadComponent/UploadComponent.tsx b/app/components/UploadComponent/UploadComponent.tsx index d84782de4..b54c86607 100644 --- a/app/components/UploadComponent/UploadComponent.tsx +++ b/app/components/UploadComponent/UploadComponent.tsx @@ -112,7 +112,7 @@ function UploadComponent({ <>
- +
{ +function toTimeString(timecode: number):number { + return Math.round(timecode * 100) / 100; +} + +export const WaveformPlayer = ({ + hasPlayButton, + onLoad, + audio +}: { + hasPlayButton?: boolean; + onLoad?:({duration}:{duration:number})=>void; + audio: string; +}) => { useSignals(); const waveSurferRef = useRef(undefined); - const [isPlaying, toggleIsPlaying] = useState(false); + const [isPlaying, toggleIsPlaying] = useState(false); + const [currenttime, setCurrenttime] = useState(0); const containerRef = useCallback((node: HTMLDivElement) => { if (node) { @@ -28,6 +41,7 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => { waveSurfer.load(newUrl); waveSurfer.on("ready", () => { waveSurferRef.current = waveSurfer; + if(onLoad) onLoad({duration: waveSurfer.getDuration()}); }); waveSurfer.on("play", () => { toggleIsPlaying(true); @@ -35,9 +49,6 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => { waveSurfer.on("pause", () => { toggleIsPlaying(false); }); - waveSurfer.on("finish", () => { - waveSurfer.seekTo(0); - }); } }, []); @@ -48,16 +59,20 @@ export const WaveformPlayer = ({ audio }: { audio: string }) => { }; }, []); + const duration = toTimeString(waveSurferRef.current ? waveSurferRef.current.getDuration() : 0); + return (
- { - waveSurferRef.current?.playPause(); - }} - /> -
+ {hasPlayButton && + { + waveSurferRef.current?.playPause(); + }} + /> + } +
); }; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx index 4055a5680..8f86fa93e 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/AudioTab.tsx @@ -21,10 +21,10 @@ import { import { PageLibrary } from "./pageLibrary"; import { PageAudioGeneration } from "./pageAudioGeneration"; -import { AudioTabPages, AudioPanelState, TtsState, V2VState } from "./types"; +import { AudioTabPages, AudioPanelState } from "./types"; import { initialTtsState, initialV2VState } from "./values"; import { TtsModelListItem } from "~/pages/PageEnigma/models/tts"; -import { VoiceConversionModelListItem } from "./typesImported"; +import { VoiceConversionModelListItem, MediaFile } from "./typesImported"; import { PageSelectTtsModel } from "./pageSelectTtsModel"; import { PageSelectV2VModel } from "./pageSelectV2VModel"; @@ -69,7 +69,12 @@ export const AudioTab = () => { })) return findNewItem !== undefined; } - ListAudioByUser(username, sessionToken).then((res:any[])=>{ + function getLength(item:any){ + return item.maybe_duration_millis + ? item.maybe_duration_millis / 1000 * 60 + : undefined; + } + ListAudioByUser(username, sessionToken).then((res:MediaFile[])=>{ audioItemsFromServer.value = res.map(item=>{ const morphedItem:AudioMediaItem = { version: 1, @@ -78,9 +83,9 @@ export const AudioTab = () => { media_id: item.token, object_uuid: item.token, name: getTitle(item), - description: item.maybe_text_transcript, + description: item.maybe_text_transcript || "", publicBucketPath: item.public_bucket_path, - length: 25, + length: getLength(item), thumbnail: "/resources/placeholders/audio_placeholder.png", isMine: true, isNew: checkIsNew(item.token), diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx index 1c86d4f49..2c98881a4 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/audioItemElement.tsx @@ -13,7 +13,11 @@ import { import { H5, H6 } from "~/components"; import { AudioTypePill } from "./audioTypePills" -import { cancelNewFromAudioItem } from "~/pages/PageEnigma/store/mediaFromServer"; +import { + cancelNewFromAudioItem, + updateAudioItemLength, + updateDemoAudioItemLength, +} from "~/pages/PageEnigma/store"; function getGcsUrl(bucketRelativePath: string | undefined | null): string { let bucket = "vocodes-public"; @@ -63,7 +67,6 @@ export const AudioItemElement = ({ item }: Props) => { const onPointerDown = useCallback( (event: React.PointerEvent) => { if(item.isNew) { - console.log('pointerdown triggered'); cancelNewFromAudioItem(item.media_id); } if (event.button === 0) { @@ -89,11 +92,27 @@ export const AudioItemElement = ({ item }: Props) => {
+ {/*

{item.length}

*/} {item.isNew &&
New*
}
{item.publicBucketPath && ( - + { + // only do this for items that doesn't have a length + if(item.category === "demo"){ + updateDemoAudioItemLength(item.media_id, duration * 60); + } + else{ + updateAudioItemLength(item.media_id, duration * 60); + } + } + } + /> )}
{item.name}
diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts index d853e7527..15a60fb50 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTab/typesImported.ts @@ -130,6 +130,8 @@ export interface MediaFile { maybe_original_filename: string | null; maybe_creator_user: UserDetailsLight | null; maybe_prompt_token: string | null; + maybe_text_transcript: string | null; + maybe_duration_millis: number | null; cover_image: { maybe_cover_image_public_bucket_path: string | null; default_cover: { diff --git a/app/pages/PageEnigma/models/assets.ts b/app/pages/PageEnigma/models/assets.ts index 376549aa1..bb81da02a 100644 --- a/app/pages/PageEnigma/models/assets.ts +++ b/app/pages/PageEnigma/models/assets.ts @@ -7,7 +7,7 @@ export interface MediaItem { name: string; description?: string; publicBucketPath?: string; - length?: number; + length?: number; // unit of length is frames, as in FPS thumbnail?: string; imageIndex?: number; isMine?: boolean; diff --git a/app/pages/PageEnigma/store/index.ts b/app/pages/PageEnigma/store/index.ts index 07628a191..fb152d4bf 100644 --- a/app/pages/PageEnigma/store/index.ts +++ b/app/pages/PageEnigma/store/index.ts @@ -4,6 +4,7 @@ export * from "./characterGroups"; export * from "./dragAndDrop"; export * from "./hotkeys"; export * from "./media"; +export * from "./mediaFromServer"; export * from "./myMovies"; export * from "./objectGroup"; export * from "./selectedItem"; diff --git a/app/pages/PageEnigma/store/media.ts b/app/pages/PageEnigma/store/media.ts index 8ed260879..ea557810f 100644 --- a/app/pages/PageEnigma/store/media.ts +++ b/app/pages/PageEnigma/store/media.ts @@ -2,6 +2,13 @@ import { signal } from "@preact/signals-core"; import { AssetType, MediaItem, AudioMediaItem } from "~/pages/PageEnigma/models"; import * as uuid from "uuid"; +export const updateDemoAudioItemLength = (mediaId:string, duration: number)=>{ + const newList = audioItems.value.map((item)=>{ + if( item.media_id === mediaId ) item.length = duration; + return item; + }) + audioItems.value = [...newList]; +} export const animationItems = signal([ { version: 1, @@ -452,7 +459,6 @@ export const audioItems = signal([ version: 1, media_id: "m_403phjvjkbbaxxbz8y7r6qjay07mfd", type: AssetType.AUDIO, - length: 25, name: "Talk (Demo Sounds)", thumbnail: "/resources/placeholders/audio_placeholder.png", category: "demo", @@ -463,7 +469,6 @@ export const audioItems = signal([ version: 1, media_id: "m_w5nn3kjh1fbkmjrdac5b2qaba0pmyt", type: AssetType.AUDIO, - length: 25, name: "NCS Song", category: "demo", thumbnail: "/resources/placeholders/audio_placeholder.png", diff --git a/app/pages/PageEnigma/store/mediaFromServer.ts b/app/pages/PageEnigma/store/mediaFromServer.ts index 8974da4e5..661ec27f1 100644 --- a/app/pages/PageEnigma/store/mediaFromServer.ts +++ b/app/pages/PageEnigma/store/mediaFromServer.ts @@ -3,11 +3,17 @@ import { AudioMediaItem } from "~/pages/PageEnigma/models"; export const audioItemsFromServer = signal([]); export const cancelNewFromAudioItem = (mediaId:string)=>{ - console.log('function called'); const newList = audioItemsFromServer.value.map((item)=>{ if( item.media_id === mediaId ) item.isNew = false; return item; }) - console.log(newList); + audioItemsFromServer.value = [...newList]; +} + +export const updateAudioItemLength = (mediaId:string, duration: number)=>{ + const newList = audioItemsFromServer.value.map((item)=>{ + if( item.media_id === mediaId ) item.length = duration; + return item; + }) audioItemsFromServer.value = [...newList]; } \ No newline at end of file