diff --git a/app/api.tsx b/app/api.tsx index 0e0bcc77a1..09dd5d1edc 100644 --- a/app/api.tsx +++ b/app/api.tsx @@ -11,4 +11,7 @@ export const inferTts = `${apiHost}/tts/inference`; export const listTts = `${apiHost}/tts/list`; // User Media -export const listMediaByUser = (username:string) => `${apiHost}/v1/media_files/list/user/${username}`; \ No newline at end of file +export const listMediaByUser = (username:string) => `${apiHost}/v1/media_files/list/user/${username}`; + +// Any Media +export const getMediaFileByToken = (fileToken:string) => `${apiHost}/v1/media_files/file/${fileToken}`; \ No newline at end of file diff --git a/app/components/Pill.tsx b/app/components/Pill.tsx new file mode 100644 index 0000000000..66d5eefc6f --- /dev/null +++ b/app/components/Pill.tsx @@ -0,0 +1,18 @@ +import { twMerge } from "tailwind-merge" + +export const Pill = ({ + className, + children, +}:{ + className?: string, + children : React.ReactNode +})=>{ + return ( +
+ {children} +
+ ) +} \ No newline at end of file diff --git a/app/components/index.tsx b/app/components/index.tsx index 28dfc10661..2133a89322 100644 --- a/app/components/index.tsx +++ b/app/components/index.tsx @@ -12,6 +12,7 @@ import { ListDropdown } from "./ListDropdown" import { ListSearchDropdown } from "./ListSearchDropdown" import { LoadingBar } from "./LoadingBar"; import { LoadingDots, LoadingDotsBricks, LoadingDotsTyping } from "./LoadingDots"; +import { Pill } from "./Pill"; export { Button, @@ -30,4 +31,5 @@ export { LoadingDots, LoadingDotsBricks, LoadingDotsTyping, + Pill }; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/AudioTabComponent.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/AudioTabComponent.tsx index d008e41c86..d3efc45118 100644 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/AudioTabComponent.tsx +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/AudioTabComponent.tsx @@ -1,41 +1,29 @@ import { useCallback, useContext, useEffect, useState } from "react"; -import { twMerge } from "tailwind-merge"; import { useSignals, useSignalEffect } from "@preact/signals-react/runtime"; -import { faCirclePlus, faArrowsRotate } from "@fortawesome/pro-solid-svg-icons"; -import { AppUiContext } from "~/contexts/AppUiContext"; import { AuthenticationContext } from "~/contexts/Authentication"; -import { APPUI_ACTION_TYPES } from "~/reducers"; -import { audioFilter, audioItems } from "~/pages/PageEnigma/store"; -import { AssetFilterOption } from "~/pages/PageEnigma/models"; - -import { Button, ButtonIcon } from "~/components"; - import { MediaItem, AssetType } from "~/pages/PageEnigma/models"; -import { ItemElements } from "~/pages/PageEnigma/comps/SidePanelTabs/itemTabs/ItemElements"; import { audioItemsFromServer } from "~/pages/PageEnigma/store/mediaFromServer"; -import { ListAudioByUser } from "./listAudioByUser"; +import { ListAudioByUser } from "./utilities"; import { inferenceJobs } from "~/pages/PageEnigma/store/inferenceJobs"; import { JobState } from "~/hooks/useInferenceJobManager/useInferenceJobManager"; +import { PageLibrary } from "./pageLibrary"; +import { PageTTS } from "./pageTTS"; +import { AudioTabPages } from "./types"; + export const AudioTabComponent = () => { useSignals(); const [ state, setState ] = useState({ firstLoad: false, fetchingUserAudio: false, + page: AudioTabPages.LIBRARY, }); const { authState } = useContext(AuthenticationContext); - const [, dispatchAppUiState] = useContext(AppUiContext); - const handleOpenTtsDialogue = ()=>{ - dispatchAppUiState({ - type: APPUI_ACTION_TYPES.OPEN_DIALOGUE_TTS - }) - } - const allAudioItems = [...audioItems.value, ...audioItemsFromServer.value]; - const handleListAudioByUser = useCallback((username:string)=>{ + const handleListAudioByUser = useCallback((username:string, sessionToken:string)=>{ setState((curr)=>({...curr, fetchingUserAudio:true})); - ListAudioByUser(username).then((res:any[])=>{ + ListAudioByUser(username, sessionToken).then((res:any[])=>{ setState((curr)=>({...curr, fetchingUserAudio:false})); audioItemsFromServer.value = res.map(item=>{ const morphedItem:MediaItem = { @@ -44,6 +32,8 @@ export const AudioTabComponent = () => { media_id: item.token, object_uuid: item.token, name: item.maybe_title || item.origin.maybe_model.title, + description: item.maybe_text_transcript, + publicBucketPath: item.public_bucket_path, length: 25, thumbnail: "/resources/placeholders/audio_placeholder.png", isMine: true, @@ -55,9 +45,9 @@ export const AudioTabComponent = () => { }, []); useEffect(()=>{ - if ( authState.userInfo ){ + if (authState.userInfo && authState.sessionToken){ if(state.firstLoad === false && audioItemsFromServer.value.length === 0){ - handleListAudioByUser(authState.userInfo.username); + handleListAudioByUser(authState.userInfo.username, authState.sessionToken); setState((curr)=>({...curr, firstLoad:true})); } } @@ -82,80 +72,30 @@ export const AudioTabComponent = () => { return foundItemOfJob !== undefined; } }); - if(found === undefined){ - handleListAudioByUser(authState.userInfo.username); + if(found === undefined && authState.sessionToken){ + handleListAudioByUser(authState.userInfo.username, authState.sessionToken); } } }); - return ( - <> -
-
- - - - { - if(authState.userInfo?.username){ - handleListAudioByUser(authState.userInfo?.username); - } - }} - /> -
-
-
- -
-
- -
- - ); + const changePage = (newPage:AudioTabPages)=>{ + setState((curr)=>({ + ...curr, + page: newPage + })) + } + if(state.page === AudioTabPages.LIBRARY){ + return + }else if(state.page === AudioTabPages.TTS && authState.sessionToken){ + return( + + ); + }else{ + return( +

Unknown Error

+ ) + } }; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/WaveformPlayer.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/WaveformPlayer.tsx new file mode 100644 index 0000000000..1603b8277f --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/WaveformPlayer.tsx @@ -0,0 +1,54 @@ +import { useCallback, useEffect, useState, useRef } from 'react'; +import WaveSurfer from 'wavesurfer.js'; +import { faCirclePlay, faCirclePause, } from "@fortawesome/pro-solid-svg-icons" +import { ButtonIcon } from '~/components'; + +export const WaveformPlayer = ({ + audio +}:{ + audio: string +}) => { + const waveSurferRef = useRef(undefined); + const [isPlaying, toggleIsPlaying] = useState(false); + + const containerRef = useCallback((node:HTMLDivElement)=>{ + if(node){ + const waveSurfer = WaveSurfer.create({ + container: node, + barWidth: 2, + height: 30, + cursorWidth: 0, + }); + waveSurfer.load(audio); + waveSurfer.on('ready', () => { + waveSurferRef.current = waveSurfer; + }); + waveSurfer.on('play', () => { + toggleIsPlaying(true); + }); + waveSurfer.on('pause', () => { + toggleIsPlaying(false); + }); + } + }, []); + + useEffect(()=>{ + return()=>{ + console.log("OK?"); + waveSurferRef.current?.destroy(); + } + },[]); + + return ( +
+ { + waveSurferRef.current?.playPause(); + }} + /> +
+
+ ); +}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/audioItemElement.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/audioItemElement.tsx new file mode 100644 index 0000000000..9ea1cf7a4b --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/audioItemElement.tsx @@ -0,0 +1,99 @@ +import { useCallback, useContext, useEffect } from "react"; +import { useSignals } from "@preact/signals-react/runtime"; + +import { WaveformPlayer } from "./WaveformPlayer"; +import { MediaItem } from "~/pages/PageEnigma/models"; +import { TrackContext } from "~/pages/PageEnigma/contexts/TrackContext/TrackContext"; +import { + canDrop, + currPosition, + dragItem, + initPosition, +} from "~/pages/PageEnigma/store"; + +import { H4, H6, Pill } from "~/components"; + +function getGcsUrl(bucketRelativePath: string | undefined | null): string { + let bucket = "vocodes-public"; + let path = bucketRelativePath; + if (path !== undefined && path !== null && !path.startsWith("/")) { + path = "/" + path; + } + return `https://storage.googleapis.com/${bucket}${path}`; +} + +interface Props { + item: MediaItem; +} + +export const AudioItemElement = ({ item }: Props) => { + useSignals(); + const { startDrag, endDrag } = useContext(TrackContext); + + const { initX, initY } = initPosition.value; + + useEffect(() => { + const onPointerUp = () => { + if (dragItem.value) { + endDrag(); + } + }; + + const onMouseMove = (event: MouseEvent) => { + if (dragItem.value) { + event.stopPropagation(); + event.preventDefault(); + const deltaX = event.pageX - initX; + const deltaY = event.pageY - initY; + currPosition.value = { currX: initX + deltaX, currY: initY + deltaY }; + } + }; + + window.addEventListener("pointerup", onPointerUp); + window.addEventListener("pointermove", onMouseMove); + + return () => { + window.removeEventListener("pointerup", onPointerUp); + window.removeEventListener("pointermove", onMouseMove); + }; + }, [startDrag, endDrag, initX, initY]); + + const onPointerDown = useCallback( + (event: React.PointerEvent) => { + if (event.button === 0) { + startDrag(item); + currPosition.value = { + currX: event.pageX, + currY: event.pageY, + }; + initPosition.value = { + initX: event.pageX, + initY: event.pageY, + }; + canDrop.value = false; + } + }, + [item, startDrag], + ); + + return ( +
+
+
+ Audio +
+ + {item.publicBucketPath && } + + +

{item.name}

+ {item.description &&
{item.description}
} +
+
+ ); +}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/audioItemElements.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/audioItemElements.tsx new file mode 100644 index 0000000000..428bb244f9 --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/audioItemElements.tsx @@ -0,0 +1,28 @@ +import { AudioItemElement } from "./audioItemElement"; +import { AssetFilterOption, MediaItem } from "~/pages/PageEnigma/models"; + +interface Props { + items: MediaItem[]; + assetFilter: AssetFilterOption; +} + +export const AudioItemElements = ({ items, assetFilter }: Props) => { + + const displayItems = items.filter((item) => { + if (assetFilter === AssetFilterOption.ALL) { + return true; + } + if (assetFilter === AssetFilterOption.MINE) { + return item.isMine; + } + return item.isBookmarked; + }); + + return ( +
+ {displayItems.map((item) => ( + + ))} +
+ ); +}; diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/listAudioByUser.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/listAudioByUser.tsx deleted file mode 100644 index 19e0d23c3e..0000000000 --- a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/listAudioByUser.tsx +++ /dev/null @@ -1,55 +0,0 @@ -import { listMediaByUser } from "~/api"; -import { - MediaFileType, - MediaFileClass, - MediaFileSubtype, - UserDetailsLight, - WeightCategory, - WeightType -} from "./types"; - - -export interface MediaFile { - token: string; - media_type: MediaFileType; - media_class: MediaFileClass | null; - maybe_media_subtype: MediaFileSubtype | null; - public_bucket_path: string; - maybe_engine_extension: string | null; - maybe_batch_token: string; - maybe_title: string | null; - maybe_original_filename: string | null; - maybe_creator_user: UserDetailsLight | null; - maybe_prompt_token: string | null; - creator_set_visibility: string; - created_at: Date; - updated_at: Date; - maybe_model_weight_info: { - title: string; - weight_token: string; - weight_category: WeightCategory; - weight_type: WeightType; - maybe_weight_creator: UserDetailsLight; - maybe_cover_image_public_bucket_path: string; - }; -} - -export const ListAudioByUser = async(username:string) => { - return await fetch(listMediaByUser(username),{ - method: 'GET', - headers: { - "Accept": "application/json", - }, - // credentials: 'include' - }) - .then(res => res.json()) - .then(res => { - if(res.success && res.results){ - return res.results.filter((item:MediaFile)=>item['media_type']==='audio'); - }else{ - Promise.reject(); - } - }) - .catch(e => ({ success : false })); -} - diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/pageLibrary.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/pageLibrary.tsx new file mode 100644 index 0000000000..79aab7fb6f --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/pageLibrary.tsx @@ -0,0 +1,79 @@ +import { faCirclePlus } from "@fortawesome/pro-solid-svg-icons"; +import { twMerge } from "tailwind-merge"; +import { useSignals } from "@preact/signals-react/runtime"; +import { audioFilter, audioItems } from "~/pages/PageEnigma/store"; +import { AssetFilterOption } from "~/pages/PageEnigma/models"; +import { audioItemsFromServer } from "~/pages/PageEnigma/store/mediaFromServer"; + +import { Button } from "~/components"; +import { AudioItemElements } from "./audioItemElements"; +import { AudioTabPages } from "./types"; + +export const PageLibrary = ({ + changePage +}:{ + changePage: (newPage:AudioTabPages) => void +})=>{ + useSignals(); + const allAudioItems = [...audioItems.value, ...audioItemsFromServer.value]; + + return ( + <> +
+
+ + + +
+
+
+ +
+ +
+ +
+ + ); +} \ No newline at end of file diff --git a/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/pageTTS.tsx b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/pageTTS.tsx new file mode 100644 index 0000000000..fd641f5bbf --- /dev/null +++ b/app/pages/PageEnigma/comps/SidePanelTabs/AudioTabComponent/pageTTS.tsx @@ -0,0 +1,233 @@ +import { useCallback, useEffect, useState } from "react"; +import { useSignalEffect } from "@preact/signals-react/runtime"; +import { v4 as uuidv4 } from "uuid"; + +import { + faChevronLeft, + faBrainCircuit, +} from "@fortawesome/pro-solid-svg-icons"; + +import { + Button, ButtonIcon, H2, Label, + LoadingDotsTyping, + ListSearchDropdown, + Textarea +} from "~/components"; + +import { TtsModelListItem, GenerateTtsAudioResponse } from "~/pages/PageEnigma/models/tts"; + +import { addInferenceJob, inferenceJobs} from "../../../store/inferenceJobs"; +import { JobState } from "~/hooks/useInferenceJobManager/useInferenceJobManager"; +import { ListTtsModels, GenerateTtsAudio, GetMediaFileByToken } from "./utilities"; +import { AudioTabPages } from "./types"; +import { AudioItemElement } from "./audioItemElement"; +import { MediaItem, AssetType } from "~/pages/PageEnigma/models"; + +type TtsState = { + voice: TtsModelListItem | undefined; + text: string; + hasEnqueued :boolean; + inferenceToken?: string; + inferenceJobType?: string; + hasTtsResult: boolean; +} +const initialState:TtsState = + { + voice:undefined, + text:"", + hasEnqueued:false, + inferenceJobType: undefined, + inferenceToken: undefined, + hasTtsResult:false, + }; + +export const PageTTS =({ + changePage, + sessionToken, +}:{ + changePage: (newPage:AudioTabPages) => void; + sessionToken: string; +})=>{ + const [ttsState, setTtsState] = useState(initialState); + + const [ttsResultFile, setTtsResultFile] = useState(); + + useSignalEffect(()=>{ + console.log(inferenceJobs.value); + if(ttsState.hasEnqueued && ttsState.inferenceToken){ + const found = inferenceJobs.value.find((job)=>job.job_id===ttsState.inferenceToken); + console.log(`finding: ${ttsState.inferenceToken}`); + console.log(found); + if(found?.job_status === JobState.COMPLETE_SUCCESS){ + setTtsState((curr)=>({ + ...curr, + hasTtsResult: true, + })) + GetMediaFileByToken(found.result.entity_token, sessionToken) + .then(res=>{ + console.log(res); + const morphedItem:MediaItem = { + version: 1, + type: AssetType.AUDIO, + media_id: res.media_file.token, + object_uuid: res.media_file.token, + name: ttsState.voice?.title || '', + description: ttsState.text, + publicBucketPath: res.media_file.public_bucket_path, + length: 25, + thumbnail: "/resources/placeholders/audio_placeholder.png", + isMine: true, + // isBookmarked?: boolean; + } + setTtsResultFile(morphedItem); + }) + } + } + }); + + const [ttsModels, setTtsModels] = useState>([]); + + const listModels = useCallback(async () => { + const ttsModelsLoaded = ttsModels.length > 0; + if (ttsModelsLoaded) { + return; // Already queried. + } + const models = await ListTtsModels(sessionToken); + if (models) { + setTtsModels(models); + } + }, []); + + useEffect(() => { + listModels(); + }, [listModels]); + + useEffect(()=> { + if(ttsState.hasEnqueued && ttsState.inferenceToken && ttsState.inferenceJobType){ + console.log(`tts has Enqueued`); + addInferenceJob({ + version:1, + job_id: ttsState.inferenceToken, + job_type: ttsState.inferenceJobType, + job_status: JobState.PENDING, + }) + } + },[ttsState]); + + const requestTts = useCallback( (sessionToken:string)=>{ + const modelToken = ttsState.voice ? ttsState.voice.model_token : undefined; + + if(modelToken){ + setTtsState((curr)=>({ + ...curr, + hasEnqueued: true, + inferenceToken: undefined, + inferenceJobType: undefined, + hasAudio: false, + result: undefined, + })); + + const request = { + uuid_idempotency_token: uuidv4(), + tts_model_token: modelToken, + inference_text: ttsState.text, + }; + + GenerateTtsAudio(request, sessionToken).then(( + res:GenerateTtsAudioResponse)=>{ + if(res && res.inference_job_token){ + setTtsState((curr)=>({ + ...curr, + inferenceToken: res.inference_job_token, + inferenceJobType: res.inference_job_token_type + })); + } + }); + }else{ + console.log("no voice model selected"); + } + },[ttsState]); + + const handleTextInput = ( + e: React.ChangeEvent + )=>{ + setTtsState((curr)=>({ + ...curr, + text: e.target.value, + })); + }; + + const handleOnSelect = (val:string)=>{ + console.log( val); + const voiceModel = ttsModels.find((item)=>{ + if (item.title === val) return item + }) + setTtsState((curr)=>({ + ...curr, + voice: voiceModel, + })); + } + + return( +
+
+ changePage(AudioTabPages.LIBRARY)} + /> +

Generate TTS

+
+ + {ttsModels.length > 0 && } +
+ +
+
+
+