cleaning up the code more

This commit is contained in:
Brandon Thomas committed 2022-03-02 04:00:24 -05:00
1 parent 82e52dcd9b
commit 45da1e96ab
5 files changed
+300

No files matched your search

@@ -0,0 +1,6 @@
class QueuePlayer {
}
export default QueuePlayer;
@@ -0,0 +1,117 @@
// This class encodes our websocket protocol and sequences playing of clips.
import { Howl } from 'howler';
import MediaQueue from "./composition/MediaQueue";
import ObsWebSocket from "./composition/ObsWebSocket";
class WebSocketProtocol {
twitchUsername: string;
mediaQueue: MediaQueue;
webSocket?: ObsWebSocket;
currentSound?: Howl;
constructor(twitchUsername: string) {
this.twitchUsername = twitchUsername;
this.mediaQueue = new MediaQueue();
}
start() {
this.newWebsocket();
this.beginPollingMediaQueue();
this.beginPollingAvailableAudioToPlay();
this.beginPollingWebsocket();
}
private newWebsocket() {
console.warn('creating new websocket');
if (this.webSocket) {
this.webSocket.close();
}
const that = this;
this.webSocket = new ObsWebSocket(
this.twitchUsername,
(ev: any) => that.onWebsocketError(ev),
(ev: any) => that.onWebsocketMessage(ev),
);
}
private beginPollingMediaQueue() {
setInterval(async () => {
await this.mediaQueue.queryAllPendingJobs();
}, 1000);
}
private beginPollingAvailableAudioToPlay() {
const that = this;
setInterval(() => {
this.playNextAvailableAudio();
}, 1000);
}
private beginPollingWebsocket() {
// NB: This has a direct bearing on how fast the backend responds.
// Increasing the delay will slow down the flow of events.
const that = this;
setInterval(async () => {
if (!!that.webSocket) {
that.webSocket.send('ping');
}
}, 1000);
}
private onWebsocketError(event: Event) {
console.log('wsError', event);
const that = this;
setTimeout(() => that.newWebsocket(), 1000);
}
private onWebsocketMessage(event: MessageEvent) {
const result = JSON.parse(event.data);
console.log('message', result);
if (result['response_type'] === 'TtsEvent') {
this.handleTtsWebsocketMessage(result);
}
}
private handleTtsWebsocketMessage(result: any) {
if ('tts_job_tokens' in result) {
result['tts_job_tokens'].forEach((jobToken : string) => {
this.mediaQueue.addNewTtsJobToken(jobToken);
});
}
}
private isCurrentlyPlayingAudio() {
return this.currentSound !== undefined && this.currentSound.playing();
}
private playNextAvailableAudio() {
if (this.isCurrentlyPlayingAudio()) {
return; // Not ready.
}
const maybePlayableAudio = this.mediaQueue.getNextAvailable();
if (!maybePlayableAudio) {
return; // No audio available.
}
const that = this;
const sound = new Howl({
src: [maybePlayableAudio.getUrl()],
onend: function(soundId: number) {
that.playNextAvailableAudio(); // eagerly play the next one.
}
});
sound.play();
}
}
export default WebSocketProtocol;
@@ -0,0 +1,94 @@
import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
import { GetTtsInferenceJobStatus, GetTtsInferenceJobStatusIsOk, TtsInferenceJobStatus } from "@storyteller/components/src/api/jobs/GetTtsInferenceJobStatus";
import { JobState, jobStateCanChange, jobStateFromString } from "@storyteller/components/src/jobs/JobStates";
import { TtsInferenceJob } from "@storyteller/components/src/jobs/TtsInferenceJobs";
import PlayableAudioFile from "./PlayableAudioFile";
enum PollingOutcome {
KeepPolling,
RemoveFromPendingQueue,
}
class MediaQueue {
// These are newly enqueued TTS jobs.
private enqueuedJobQueue: TtsInferenceJob[];
// This is audio that is ready to play!
private readyToPlayAudioQueue: PlayableAudioFile[];
constructor() {
this.enqueuedJobQueue = [];
this.readyToPlayAudioQueue = [];
}
// Add a new TTS job
addNewTtsJobToken(jobToken: string) {
const newJob = new TtsInferenceJob(jobToken);
this.enqueuedJobQueue.push(newJob);
}
// Get the next audio file that is available (if it exists)
getNextAvailable() : PlayableAudioFile | undefined {
return this.readyToPlayAudioQueue.shift();
}
// Query all jobs *ONCE*
async queryAllPendingJobs() {
let removeJobTokens = new Set();
for (let job of this.enqueuedJobQueue) {
if (jobStateCanChange(job.jobState)) {
let pollingOutcome = await this.queryPendingJobForStatus(job.jobToken);
if (pollingOutcome === PollingOutcome.RemoveFromPendingQueue) {
removeJobTokens.add(job.jobToken);
}
}
}
this.enqueuedJobQueue =
this.enqueuedJobQueue.filter(job => !removeJobTokens.has(job.jobToken));
}
private async queryPendingJobForStatus(jobToken: string) : Promise<PollingOutcome> {
const response = await GetTtsInferenceJobStatus(jobToken);
if (GetTtsInferenceJobStatusIsOk(response)) {
const jobState = jobStateFromString(response.state.status);
switch (jobState) {
case JobState.UNKNOWN:
case JobState.PENDING:
case JobState.STARTED:
case JobState.ATTEMPT_FAILED:
return PollingOutcome.KeepPolling;
case JobState.COMPLETE_SUCCESS:
this.addReadyToPlayJob(response.state);
return PollingOutcome.RemoveFromPendingQueue;
case JobState.COMPLETE_FAILURE:
case JobState.DEAD:
return PollingOutcome.RemoveFromPendingQueue;
}
}
return PollingOutcome.KeepPolling;
}
private addReadyToPlayJob(jobStatusResponse: TtsInferenceJobStatus) {
const wavPath = jobStatusResponse.maybe_public_bucket_wav_audio_path;
if (!wavPath) {
return;
}
const wavUrl = new BucketConfig().getGcsUrl(wavPath);
const playable = new PlayableAudioFile(wavUrl);
this.readyToPlayAudioQueue.push(playable);
}
}
export default MediaQueue;
@@ -0,0 +1,66 @@
import { ApiConfig } from "@storyteller/components";
class ObsWebSocket {
private webSocket: WebSocket;
private onErrorCallback?: (event: Event) => void;
private onMessageCallback?: (event: MessageEvent) => void;
constructor(
twitchUsername: string,
onError?: (event: Event) => void,
onMessage?: (event: MessageEvent) => void,
) {
const url = new ApiConfig().obsEventsWebsocket(twitchUsername);
const webSocket = new WebSocket(url);
if (!!onError) {
this.onErrorCallback = onError;
}
if (!!onMessage) {
this.onMessageCallback = onMessage;
}
const that = this;
webSocket.onopen = function (event: Event) { that.onOpen(event); };
webSocket.onerror = function(event: Event) { that.onError(event); };
webSocket.onmessage = function (event: MessageEvent) { that.onMessage(event); }
this.webSocket = webSocket;
}
send(data: string) {
this.webSocket.send(data);
}
close() {
this.onErrorCallback = undefined;
this.onMessageCallback = undefined;
this.webSocket.close();
}
// ========== Handlers ==========
private onOpen(event: Event) {
console.log('open ws');
}
private onMessage(event: MessageEvent) {
console.log('on message');
if (!!this.onMessageCallback) {
this.onMessageCallback(event);
}
}
private onError(event: Event) {
if (!!this.onErrorCallback) {
this.onErrorCallback(event);
}
}
}
export default ObsWebSocket;
@@ -0,0 +1,17 @@
// TODO: Add twitch user data, and lots of other stats that might be useful
// at audio play-time.
class PlayableAudioFile {
private mediaUrl: string;
constructor(mediaUrl: string) {
this.mediaUrl = mediaUrl;
}
getUrl() : string {
return this.mediaUrl;
}
}
export default PlayableAudioFile;