From 45da1e96ab11d6d69942b0b8a05c878cd1343b32 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 2 Mar 2022 04:00:24 -0500 Subject: [PATCH] cleaning up the code more --- .../pages/obs-layer/player/QueuePlayer.tsx | 6 + .../obs-layer/player/WebSocketProtocol.ts | 117 ++++++++++++++++++ .../player/composition/MediaQueue.ts | 94 ++++++++++++++ .../player/composition/ObsWebSocket.ts | 66 ++++++++++ .../player/composition/PlayableAudioFile.ts | 17 +++ 5 files changed, 300 insertions(+) create mode 100644 packages/storyteller-obs/src/pages/obs-layer/player/QueuePlayer.tsx create mode 100644 packages/storyteller-obs/src/pages/obs-layer/player/WebSocketProtocol.ts create mode 100644 packages/storyteller-obs/src/pages/obs-layer/player/composition/MediaQueue.ts create mode 100644 packages/storyteller-obs/src/pages/obs-layer/player/composition/ObsWebSocket.ts create mode 100644 packages/storyteller-obs/src/pages/obs-layer/player/composition/PlayableAudioFile.ts diff --git a/packages/storyteller-obs/src/pages/obs-layer/player/QueuePlayer.tsx b/packages/storyteller-obs/src/pages/obs-layer/player/QueuePlayer.tsx new file mode 100644 index 0000000000..e77012602f --- /dev/null +++ b/packages/storyteller-obs/src/pages/obs-layer/player/QueuePlayer.tsx @@ -0,0 +1,6 @@ + +class QueuePlayer { + +} + +export default QueuePlayer; diff --git a/packages/storyteller-obs/src/pages/obs-layer/player/WebSocketProtocol.ts b/packages/storyteller-obs/src/pages/obs-layer/player/WebSocketProtocol.ts new file mode 100644 index 0000000000..70b22ac731 --- /dev/null +++ b/packages/storyteller-obs/src/pages/obs-layer/player/WebSocketProtocol.ts @@ -0,0 +1,117 @@ +// This class encodes our websocket protocol and sequences playing of clips. + +import { Howl } from 'howler'; +import MediaQueue from "./composition/MediaQueue"; +import ObsWebSocket from "./composition/ObsWebSocket"; + +class WebSocketProtocol { + twitchUsername: string; + + mediaQueue: MediaQueue; + + webSocket?: ObsWebSocket; + + currentSound?: Howl; + + constructor(twitchUsername: string) { + this.twitchUsername = twitchUsername; + this.mediaQueue = new MediaQueue(); + } + + start() { + this.newWebsocket(); + this.beginPollingMediaQueue(); + this.beginPollingAvailableAudioToPlay(); + this.beginPollingWebsocket(); + } + + private newWebsocket() { + console.warn('creating new websocket'); + if (this.webSocket) { + this.webSocket.close(); + } + const that = this; + this.webSocket = new ObsWebSocket( + this.twitchUsername, + (ev: any) => that.onWebsocketError(ev), + (ev: any) => that.onWebsocketMessage(ev), + ); + } + + private beginPollingMediaQueue() { + setInterval(async () => { + await this.mediaQueue.queryAllPendingJobs(); + }, 1000); + } + + private beginPollingAvailableAudioToPlay() { + const that = this; + setInterval(() => { + this.playNextAvailableAudio(); + }, 1000); + } + + private beginPollingWebsocket() { + // NB: This has a direct bearing on how fast the backend responds. + // Increasing the delay will slow down the flow of events. + const that = this; + setInterval(async () => { + if (!!that.webSocket) { + that.webSocket.send('ping'); + } + }, 1000); + } + + + private onWebsocketError(event: Event) { + console.log('wsError', event); + const that = this; + setTimeout(() => that.newWebsocket(), 1000); + } + + private onWebsocketMessage(event: MessageEvent) { + const result = JSON.parse(event.data); + + console.log('message', result); + + if (result['response_type'] === 'TtsEvent') { + this.handleTtsWebsocketMessage(result); + } + } + + private handleTtsWebsocketMessage(result: any) { + if ('tts_job_tokens' in result) { + result['tts_job_tokens'].forEach((jobToken : string) => { + this.mediaQueue.addNewTtsJobToken(jobToken); + }); + } + } + + private isCurrentlyPlayingAudio() { + return this.currentSound !== undefined && this.currentSound.playing(); + } + + private playNextAvailableAudio() { + if (this.isCurrentlyPlayingAudio()) { + return; // Not ready. + } + + const maybePlayableAudio = this.mediaQueue.getNextAvailable(); + if (!maybePlayableAudio) { + return; // No audio available. + } + + const that = this; + + const sound = new Howl({ + src: [maybePlayableAudio.getUrl()], + onend: function(soundId: number) { + that.playNextAvailableAudio(); // eagerly play the next one. + } + }); + + sound.play(); + } +} + +export default WebSocketProtocol; diff --git a/packages/storyteller-obs/src/pages/obs-layer/player/composition/MediaQueue.ts b/packages/storyteller-obs/src/pages/obs-layer/player/composition/MediaQueue.ts new file mode 100644 index 0000000000..fb8004465d --- /dev/null +++ b/packages/storyteller-obs/src/pages/obs-layer/player/composition/MediaQueue.ts @@ -0,0 +1,94 @@ +import { BucketConfig } from "@storyteller/components/src/api/BucketConfig"; +import { GetTtsInferenceJobStatus, GetTtsInferenceJobStatusIsOk, TtsInferenceJobStatus } from "@storyteller/components/src/api/jobs/GetTtsInferenceJobStatus"; +import { JobState, jobStateCanChange, jobStateFromString } from "@storyteller/components/src/jobs/JobStates"; +import { TtsInferenceJob } from "@storyteller/components/src/jobs/TtsInferenceJobs"; +import PlayableAudioFile from "./PlayableAudioFile"; + +enum PollingOutcome { + KeepPolling, + RemoveFromPendingQueue, +} + +class MediaQueue { + + // These are newly enqueued TTS jobs. + private enqueuedJobQueue: TtsInferenceJob[]; + + // This is audio that is ready to play! + private readyToPlayAudioQueue: PlayableAudioFile[]; + + constructor() { + this.enqueuedJobQueue = []; + this.readyToPlayAudioQueue = []; + } + + // Add a new TTS job + addNewTtsJobToken(jobToken: string) { + const newJob = new TtsInferenceJob(jobToken); + this.enqueuedJobQueue.push(newJob); + } + + // Get the next audio file that is available (if it exists) + getNextAvailable() : PlayableAudioFile | undefined { + return this.readyToPlayAudioQueue.shift(); + } + + // Query all jobs *ONCE* + async queryAllPendingJobs() { + let removeJobTokens = new Set(); + + for (let job of this.enqueuedJobQueue) { + if (jobStateCanChange(job.jobState)) { + let pollingOutcome = await this.queryPendingJobForStatus(job.jobToken); + + if (pollingOutcome === PollingOutcome.RemoveFromPendingQueue) { + removeJobTokens.add(job.jobToken); + } + } + } + + this.enqueuedJobQueue = + this.enqueuedJobQueue.filter(job => !removeJobTokens.has(job.jobToken)); + } + + private async queryPendingJobForStatus(jobToken: string) : Promise { + const response = await GetTtsInferenceJobStatus(jobToken); + if (GetTtsInferenceJobStatusIsOk(response)) { + + const jobState = jobStateFromString(response.state.status); + + switch (jobState) { + case JobState.UNKNOWN: + case JobState.PENDING: + case JobState.STARTED: + case JobState.ATTEMPT_FAILED: + return PollingOutcome.KeepPolling; + + case JobState.COMPLETE_SUCCESS: + this.addReadyToPlayJob(response.state); + return PollingOutcome.RemoveFromPendingQueue; + + case JobState.COMPLETE_FAILURE: + case JobState.DEAD: + return PollingOutcome.RemoveFromPendingQueue; + } + } + + return PollingOutcome.KeepPolling; + } + + private addReadyToPlayJob(jobStatusResponse: TtsInferenceJobStatus) { + const wavPath = jobStatusResponse.maybe_public_bucket_wav_audio_path; + + if (!wavPath) { + return; + } + + const wavUrl = new BucketConfig().getGcsUrl(wavPath); + const playable = new PlayableAudioFile(wavUrl); + + this.readyToPlayAudioQueue.push(playable); + } +} + +export default MediaQueue; diff --git a/packages/storyteller-obs/src/pages/obs-layer/player/composition/ObsWebSocket.ts b/packages/storyteller-obs/src/pages/obs-layer/player/composition/ObsWebSocket.ts new file mode 100644 index 0000000000..fea0879335 --- /dev/null +++ b/packages/storyteller-obs/src/pages/obs-layer/player/composition/ObsWebSocket.ts @@ -0,0 +1,66 @@ +import { ApiConfig } from "@storyteller/components"; + +class ObsWebSocket { + + private webSocket: WebSocket; + + private onErrorCallback?: (event: Event) => void; + private onMessageCallback?: (event: MessageEvent) => void; + + constructor( + twitchUsername: string, + onError?: (event: Event) => void, + onMessage?: (event: MessageEvent) => void, + ) { + const url = new ApiConfig().obsEventsWebsocket(twitchUsername); + const webSocket = new WebSocket(url); + + if (!!onError) { + this.onErrorCallback = onError; + } + + if (!!onMessage) { + this.onMessageCallback = onMessage; + } + + const that = this; + + webSocket.onopen = function (event: Event) { that.onOpen(event); }; + webSocket.onerror = function(event: Event) { that.onError(event); }; + webSocket.onmessage = function (event: MessageEvent) { that.onMessage(event); } + + this.webSocket = webSocket; + } + + send(data: string) { + this.webSocket.send(data); + } + + close() { + this.onErrorCallback = undefined; + this.onMessageCallback = undefined; + this.webSocket.close(); + } + + // ========== Handlers ========== + + private onOpen(event: Event) { + + console.log('open ws'); + } + + private onMessage(event: MessageEvent) { + console.log('on message'); + if (!!this.onMessageCallback) { + this.onMessageCallback(event); + } + } + + private onError(event: Event) { + if (!!this.onErrorCallback) { + this.onErrorCallback(event); + } + } +} + +export default ObsWebSocket; \ No newline at end of file diff --git a/packages/storyteller-obs/src/pages/obs-layer/player/composition/PlayableAudioFile.ts b/packages/storyteller-obs/src/pages/obs-layer/player/composition/PlayableAudioFile.ts new file mode 100644 index 0000000000..7e4d3951f6 --- /dev/null +++ b/packages/storyteller-obs/src/pages/obs-layer/player/composition/PlayableAudioFile.ts @@ -0,0 +1,17 @@ + +// TODO: Add twitch user data, and lots of other stats that might be useful +// at audio play-time. +class PlayableAudioFile { + + private mediaUrl: string; + + constructor(mediaUrl: string) { + this.mediaUrl = mediaUrl; + } + + getUrl() : string { + return this.mediaUrl; + } +} + +export default PlayableAudioFile;