mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-18 07:35:19 +00:00
cleaning up the code more
This commit is contained in:
1 parent
82e52dcd9b
commit
45da1e96ab
5 files changed
+300
No files matched your search
@@ -0,0 +1,6 @@
|
||||
|
||||
class QueuePlayer {
|
||||
|
||||
}
|
||||
|
||||
export default QueuePlayer;
|
||||
@@ -0,0 +1,117 @@
|
||||
// This class encodes our websocket protocol and sequences playing of clips.
|
||||
|
||||
import { Howl } from 'howler';
|
||||
import MediaQueue from "./composition/MediaQueue";
|
||||
import ObsWebSocket from "./composition/ObsWebSocket";
|
||||
|
||||
class WebSocketProtocol {
|
||||
twitchUsername: string;
|
||||
|
||||
mediaQueue: MediaQueue;
|
||||
|
||||
webSocket?: ObsWebSocket;
|
||||
|
||||
currentSound?: Howl;
|
||||
|
||||
constructor(twitchUsername: string) {
|
||||
this.twitchUsername = twitchUsername;
|
||||
this.mediaQueue = new MediaQueue();
|
||||
}
|
||||
|
||||
start() {
|
||||
this.newWebsocket();
|
||||
this.beginPollingMediaQueue();
|
||||
this.beginPollingAvailableAudioToPlay();
|
||||
this.beginPollingWebsocket();
|
||||
}
|
||||
|
||||
private newWebsocket() {
|
||||
console.warn('creating new websocket');
|
||||
if (this.webSocket) {
|
||||
this.webSocket.close();
|
||||
}
|
||||
const that = this;
|
||||
this.webSocket = new ObsWebSocket(
|
||||
this.twitchUsername,
|
||||
(ev: any) => that.onWebsocketError(ev),
|
||||
(ev: any) => that.onWebsocketMessage(ev),
|
||||
);
|
||||
}
|
||||
|
||||
private beginPollingMediaQueue() {
|
||||
setInterval(async () => {
|
||||
await this.mediaQueue.queryAllPendingJobs();
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
private beginPollingAvailableAudioToPlay() {
|
||||
const that = this;
|
||||
setInterval(() => {
|
||||
this.playNextAvailableAudio();
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
private beginPollingWebsocket() {
|
||||
// NB: This has a direct bearing on how fast the backend responds.
|
||||
// Increasing the delay will slow down the flow of events.
|
||||
const that = this;
|
||||
setInterval(async () => {
|
||||
if (!!that.webSocket) {
|
||||
that.webSocket.send('ping');
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
|
||||
private onWebsocketError(event: Event) {
|
||||
console.log('wsError', event);
|
||||
const that = this;
|
||||
setTimeout(() => that.newWebsocket(), 1000);
|
||||
}
|
||||
|
||||
private onWebsocketMessage(event: MessageEvent) {
|
||||
const result = JSON.parse(event.data);
|
||||
|
||||
console.log('message', result);
|
||||
|
||||
if (result['response_type'] === 'TtsEvent') {
|
||||
this.handleTtsWebsocketMessage(result);
|
||||
}
|
||||
}
|
||||
|
||||
private handleTtsWebsocketMessage(result: any) {
|
||||
if ('tts_job_tokens' in result) {
|
||||
result['tts_job_tokens'].forEach((jobToken : string) => {
|
||||
this.mediaQueue.addNewTtsJobToken(jobToken);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private isCurrentlyPlayingAudio() {
|
||||
return this.currentSound !== undefined && this.currentSound.playing();
|
||||
}
|
||||
|
||||
private playNextAvailableAudio() {
|
||||
if (this.isCurrentlyPlayingAudio()) {
|
||||
return; // Not ready.
|
||||
}
|
||||
|
||||
const maybePlayableAudio = this.mediaQueue.getNextAvailable();
|
||||
if (!maybePlayableAudio) {
|
||||
return; // No audio available.
|
||||
}
|
||||
|
||||
const that = this;
|
||||
|
||||
const sound = new Howl({
|
||||
src: [maybePlayableAudio.getUrl()],
|
||||
onend: function(soundId: number) {
|
||||
that.playNextAvailableAudio(); // eagerly play the next one.
|
||||
}
|
||||
});
|
||||
|
||||
sound.play();
|
||||
}
|
||||
}
|
||||
|
||||
export default WebSocketProtocol;
|
||||
@@ -0,0 +1,94 @@
|
||||
import { BucketConfig } from "@storyteller/components/src/api/BucketConfig";
|
||||
import { GetTtsInferenceJobStatus, GetTtsInferenceJobStatusIsOk, TtsInferenceJobStatus } from "@storyteller/components/src/api/jobs/GetTtsInferenceJobStatus";
|
||||
import { JobState, jobStateCanChange, jobStateFromString } from "@storyteller/components/src/jobs/JobStates";
|
||||
import { TtsInferenceJob } from "@storyteller/components/src/jobs/TtsInferenceJobs";
|
||||
import PlayableAudioFile from "./PlayableAudioFile";
|
||||
|
||||
enum PollingOutcome {
|
||||
KeepPolling,
|
||||
RemoveFromPendingQueue,
|
||||
}
|
||||
|
||||
class MediaQueue {
|
||||
|
||||
// These are newly enqueued TTS jobs.
|
||||
private enqueuedJobQueue: TtsInferenceJob[];
|
||||
|
||||
// This is audio that is ready to play!
|
||||
private readyToPlayAudioQueue: PlayableAudioFile[];
|
||||
|
||||
constructor() {
|
||||
this.enqueuedJobQueue = [];
|
||||
this.readyToPlayAudioQueue = [];
|
||||
}
|
||||
|
||||
// Add a new TTS job
|
||||
addNewTtsJobToken(jobToken: string) {
|
||||
const newJob = new TtsInferenceJob(jobToken);
|
||||
this.enqueuedJobQueue.push(newJob);
|
||||
}
|
||||
|
||||
// Get the next audio file that is available (if it exists)
|
||||
getNextAvailable() : PlayableAudioFile | undefined {
|
||||
return this.readyToPlayAudioQueue.shift();
|
||||
}
|
||||
|
||||
// Query all jobs *ONCE*
|
||||
async queryAllPendingJobs() {
|
||||
let removeJobTokens = new Set();
|
||||
|
||||
for (let job of this.enqueuedJobQueue) {
|
||||
if (jobStateCanChange(job.jobState)) {
|
||||
let pollingOutcome = await this.queryPendingJobForStatus(job.jobToken);
|
||||
|
||||
if (pollingOutcome === PollingOutcome.RemoveFromPendingQueue) {
|
||||
removeJobTokens.add(job.jobToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.enqueuedJobQueue =
|
||||
this.enqueuedJobQueue.filter(job => !removeJobTokens.has(job.jobToken));
|
||||
}
|
||||
|
||||
private async queryPendingJobForStatus(jobToken: string) : Promise<PollingOutcome> {
|
||||
const response = await GetTtsInferenceJobStatus(jobToken);
|
||||
if (GetTtsInferenceJobStatusIsOk(response)) {
|
||||
|
||||
const jobState = jobStateFromString(response.state.status);
|
||||
|
||||
switch (jobState) {
|
||||
case JobState.UNKNOWN:
|
||||
case JobState.PENDING:
|
||||
case JobState.STARTED:
|
||||
case JobState.ATTEMPT_FAILED:
|
||||
return PollingOutcome.KeepPolling;
|
||||
|
||||
case JobState.COMPLETE_SUCCESS:
|
||||
this.addReadyToPlayJob(response.state);
|
||||
return PollingOutcome.RemoveFromPendingQueue;
|
||||
|
||||
case JobState.COMPLETE_FAILURE:
|
||||
case JobState.DEAD:
|
||||
return PollingOutcome.RemoveFromPendingQueue;
|
||||
}
|
||||
}
|
||||
|
||||
return PollingOutcome.KeepPolling;
|
||||
}
|
||||
|
||||
private addReadyToPlayJob(jobStatusResponse: TtsInferenceJobStatus) {
|
||||
const wavPath = jobStatusResponse.maybe_public_bucket_wav_audio_path;
|
||||
|
||||
if (!wavPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wavUrl = new BucketConfig().getGcsUrl(wavPath);
|
||||
const playable = new PlayableAudioFile(wavUrl);
|
||||
|
||||
this.readyToPlayAudioQueue.push(playable);
|
||||
}
|
||||
}
|
||||
|
||||
export default MediaQueue;
|
||||
@@ -0,0 +1,66 @@
|
||||
import { ApiConfig } from "@storyteller/components";
|
||||
|
||||
class ObsWebSocket {
|
||||
|
||||
private webSocket: WebSocket;
|
||||
|
||||
private onErrorCallback?: (event: Event) => void;
|
||||
private onMessageCallback?: (event: MessageEvent) => void;
|
||||
|
||||
constructor(
|
||||
twitchUsername: string,
|
||||
onError?: (event: Event) => void,
|
||||
onMessage?: (event: MessageEvent) => void,
|
||||
) {
|
||||
const url = new ApiConfig().obsEventsWebsocket(twitchUsername);
|
||||
const webSocket = new WebSocket(url);
|
||||
|
||||
if (!!onError) {
|
||||
this.onErrorCallback = onError;
|
||||
}
|
||||
|
||||
if (!!onMessage) {
|
||||
this.onMessageCallback = onMessage;
|
||||
}
|
||||
|
||||
const that = this;
|
||||
|
||||
webSocket.onopen = function (event: Event) { that.onOpen(event); };
|
||||
webSocket.onerror = function(event: Event) { that.onError(event); };
|
||||
webSocket.onmessage = function (event: MessageEvent) { that.onMessage(event); }
|
||||
|
||||
this.webSocket = webSocket;
|
||||
}
|
||||
|
||||
send(data: string) {
|
||||
this.webSocket.send(data);
|
||||
}
|
||||
|
||||
close() {
|
||||
this.onErrorCallback = undefined;
|
||||
this.onMessageCallback = undefined;
|
||||
this.webSocket.close();
|
||||
}
|
||||
|
||||
// ========== Handlers ==========
|
||||
|
||||
private onOpen(event: Event) {
|
||||
|
||||
console.log('open ws');
|
||||
}
|
||||
|
||||
private onMessage(event: MessageEvent) {
|
||||
console.log('on message');
|
||||
if (!!this.onMessageCallback) {
|
||||
this.onMessageCallback(event);
|
||||
}
|
||||
}
|
||||
|
||||
private onError(event: Event) {
|
||||
if (!!this.onErrorCallback) {
|
||||
this.onErrorCallback(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default ObsWebSocket;
|
||||
@@ -0,0 +1,17 @@
|
||||
|
||||
// TODO: Add twitch user data, and lots of other stats that might be useful
|
||||
// at audio play-time.
|
||||
class PlayableAudioFile {
|
||||
|
||||
private mediaUrl: string;
|
||||
|
||||
constructor(mediaUrl: string) {
|
||||
this.mediaUrl = mediaUrl;
|
||||
}
|
||||
|
||||
getUrl() : string {
|
||||
return this.mediaUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export default PlayableAudioFile;
|
||||
Reference in new issue
Block a user