{
+
+ constructor(props: Props) {
+ super(props);
+ }
+
+ public render() {
+ return (
+
+ Select Speaker
+
+
+
+ );
+ }
+}
+
+export {SpeakerDropdownComponent};
diff --git a/frontends/twitch-early-access/src/components/speaker_mode/SpeakerModeComponent.tsx b/frontends/twitch-early-access/src/components/speaker_mode/SpeakerModeComponent.tsx
new file mode 100644
index 0000000000..8513aa54bc
--- /dev/null
+++ b/frontends/twitch-early-access/src/components/speaker_mode/SpeakerModeComponent.tsx
@@ -0,0 +1,81 @@
+import React from 'react';
+import { SpeakerDetails, SpeakerResponse } from './SpeakerResponse';
+import { SpeakerDropdownComponent } from './SpeakerDropdownComponent';
+import SpeakerAudioForm from './SpeakerAudioForm';
+import ApiConfig from '../../api/ApiConfig';
+
+interface Props {
+ apiConfig: ApiConfig,
+ text: string,
+ updateTextCallback: (text: string) => void,
+}
+
+interface State {
+ speaker_response?: SpeakerResponse,
+ speaker_slug?: string,
+}
+
+class SpeakerModeComponent extends React.Component {
+
+ constructor(props: Props) {
+ super(props);
+ this.state = {
+ speaker_response: undefined,
+ speaker_slug: undefined,
+ };
+ }
+
+ componentDidMount() {
+ this.loadSpeakers();
+ }
+
+ public loadSpeakers() {
+ const url = this.props.apiConfig.getEndpoint('/voices');
+ fetch(url)
+ .then(res => res.json())
+ .then(
+ (result) => {
+ console.log('loaded speakers', result);
+ let first_speaker = result.speakers[0].slug;
+ this.setState({
+ speaker_response: result,
+ speaker_slug: first_speaker,
+ })
+ }
+ );
+ }
+
+ changeSpeaker = (ev: React.FormEvent) => {
+ const speakerSlug = (ev.target as HTMLInputElement).value;
+ this.setState({
+ speaker_slug: speakerSlug,
+ })
+ }
+
+ public render() {
+ let speakers = this.state.speaker_response?.speakers;
+ let dropdown;
+
+ if (speakers?.length || 0 > 0) {
+ dropdown =
+ }
+
+ return (
+
+ SpeakerMode
+ {dropdown}
+
+
+ );
+ }
+}
+
+export {SpeakerModeComponent};
diff --git a/frontends/twitch-early-access/src/components/speaker_mode/SpeakerResponse.ts b/frontends/twitch-early-access/src/components/speaker_mode/SpeakerResponse.ts
new file mode 100644
index 0000000000..e6532c35de
--- /dev/null
+++ b/frontends/twitch-early-access/src/components/speaker_mode/SpeakerResponse.ts
@@ -0,0 +1,15 @@
+
+
+class SpeakerResponse {
+ speakers?: SpeakerDetails[]
+}
+
+class SpeakerDetails {
+ name?: string
+ slug?: string
+ model_pipeline?: string
+ tacotron?: string
+ melgan?: string
+}
+
+export {SpeakerResponse, SpeakerDetails};
diff --git a/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerDropdownComponent.tsx b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerDropdownComponent.tsx
new file mode 100644
index 0000000000..5d7d657e30
--- /dev/null
+++ b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerDropdownComponent.tsx
@@ -0,0 +1,36 @@
+
+import React from 'react';
+import { SpeakerDetails } from './SpeakerResponse';
+
+interface Props {
+ changeCallback: (ev: React.FormEvent) => void,
+ speakers: SpeakerDetails[]
+}
+
+interface State {
+}
+
+class SpeakerDropdownComponent extends React.Component {
+
+ constructor(props: Props) {
+ super(props);
+ }
+
+ public render() {
+ return (
+
+ Select Speaker
+
+
+
+ );
+ }
+}
+
+export {SpeakerDropdownComponent};
diff --git a/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerResponse.ts b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerResponse.ts
new file mode 100644
index 0000000000..e6532c35de
--- /dev/null
+++ b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerResponse.ts
@@ -0,0 +1,15 @@
+
+
+class SpeakerResponse {
+ speakers?: SpeakerDetails[]
+}
+
+class SpeakerDetails {
+ name?: string
+ slug?: string
+ model_pipeline?: string
+ tacotron?: string
+ melgan?: string
+}
+
+export {SpeakerResponse, SpeakerDetails};
diff --git a/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramAudioForm.tsx b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramAudioForm.tsx
new file mode 100644
index 0000000000..08a600ee82
--- /dev/null
+++ b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramAudioForm.tsx
@@ -0,0 +1,175 @@
+import React from 'react';
+import Howl from 'howler';
+import ApiConfig from '../../api/ApiConfig';
+import { createColorMap, linearScale } from "@colormap/core";
+import { magma } from "@colormap/presets";
+import { SpeakRequest } from '../../api/ApiDefinition'
+
+interface Props {
+ apiConfig: ApiConfig,
+ speaker?: String,
+ text: string,
+ updateTextCallback: (text: string) => void,
+}
+
+interface State {
+ howl?: Howl.Howl,
+ spectrogram?: Spectrogram,
+}
+
+class Spectrogram {
+ pixels: Uint8ClampedArray;
+ width: number;
+ height: number;
+
+ constructor(pixels: Uint8ClampedArray, width: number, height: number) {
+ this.pixels = pixels;
+ this.width = width;
+ this.height = height;
+ }
+}
+
+class SpeakerSpectrogramAudioForm extends React.Component {
+
+ constructor(props: Props) {
+ super(props);
+ this.state = {
+ howl: undefined,
+ spectrogram: undefined,
+ };
+ }
+
+ handleTextChange = (ev: React.FormEvent) => {
+ const text = (ev.target as HTMLInputElement).value;
+ this.props.updateTextCallback(text);
+ }
+
+ makeRequest = (ev: React.FormEvent) => {
+ console.log("Form Submit");
+
+ if (!this.props.text) {
+ return;
+ }
+
+ let request = new SpeakRequest(this.props.text, this.props.speaker!);
+
+ const url = this.props.apiConfig.getEndpoint('/speak_spectrogram');
+
+ fetch(url, {
+ method: 'POST',
+ headers: {
+ 'Accept': 'application/json',
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify(request),
+ })
+ .then(res => res.json())
+ .then(res => {
+ const data = `data:audio/wav;base64,${res.audio_base64}`;
+
+ const audioUrl = new URL(data);
+
+ // var image = new Image();
+ // image.src = `data:image/bmp;base64,${res.spectrogram.bytes_base64}`;
+ // console.log('image', image);
+
+ let scale = linearScale([0, 255], [0, 1]);
+ let colorMap = createColorMap(magma, scale);
+
+ // https://stackoverflow.com/a/21797381
+ function base64ToArrayBuffer(base64string: string) : Uint8ClampedArray {
+ var binary_string = window.atob(base64string);
+ var len = binary_string.length * 4;
+ let bytes = new Uint8ClampedArray(len);
+ for (let i = 0, j = 0; i < len; i++, j += 4) {
+ let val = binary_string.charCodeAt(i);
+ if (isNaN(val)) {
+ val = 0;
+ }
+ let mapped = colorMap(val);
+ let r = Math.floor(mapped[0] * 255)
+ let g = Math.floor(mapped[1] * 255)
+ let b = Math.floor(mapped[2] * 255)
+ bytes[j+0] = r;
+ bytes[j+1] = g;
+ bytes[j+2] = b;
+ bytes[j+3] = 255;
+ }
+ return bytes;
+ }
+
+ let bytes = base64ToArrayBuffer(res.spectrogram.bytes_base64);
+
+ const spectrogram = new Spectrogram(bytes, res.spectrogram.width, res.spectrogram.height);
+
+ const sound = new Howl.Howl({
+ src: [data],
+ //format: 'wav',
+ });
+
+ this.setState({
+ howl: sound,
+ spectrogram: spectrogram,
+ });
+
+ sound.play();
+
+ (window as any).sound = sound;
+ });
+
+ ev.preventDefault();
+ return false;
+
+ }
+
+ componentDidMount() {
+ this.updateCanvas();
+ }
+
+ componentDidUpdate() {
+ this.updateCanvas();
+ }
+
+ updateCanvas() {
+ const ctx = (this.refs.canvas as any).getContext('2d');
+ // let width = 300;
+ // let height = 80;
+ // ctx.clearRect(0,0, width, height);
+
+ if (this.state.spectrogram !== undefined) {
+ let width = this.state.spectrogram!.width;
+ let height = this.state.spectrogram!.height;
+
+ var image = new ImageData(this.state.spectrogram!.pixels, width, height);
+
+ createImageBitmap(image).then(renderer => {
+ ctx.drawImage(renderer, 0, 0, width * 3, height * 3)
+ });
+ }
+ }
+
+ public render() {
+ let width = 150 * 3;
+ let height = 80 * 3;
+
+ if (this.state.spectrogram !== undefined) {
+ width = this.state.spectrogram.width * 3;
+ height = this.state.spectrogram.height * 3;
+ }
+
+ let canvas =
+
+ return (
+
+ {canvas}
+
+
+
+ );
+ }
+}
+
+export default SpeakerSpectrogramAudioForm;
diff --git a/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramModeComponent.tsx b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramModeComponent.tsx
new file mode 100644
index 0000000000..3fdfa19965
--- /dev/null
+++ b/frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramModeComponent.tsx
@@ -0,0 +1,81 @@
+import React from 'react';
+import { SpeakerDetails, SpeakerResponse } from './SpeakerResponse';
+import { SpeakerDropdownComponent } from './SpeakerDropdownComponent';
+import SpeakerSpectrogramAudioForm from './SpeakerSpectrogramAudioForm';
+import ApiConfig from '../../api/ApiConfig';
+
+interface Props {
+ apiConfig: ApiConfig,
+ text: string,
+ updateTextCallback: (text: string) => void,
+}
+
+interface State {
+ speaker_response?: SpeakerResponse,
+ speaker_slug?: string,
+}
+
+class SpeakerSpectrogramModeComponent extends React.Component {
+
+ constructor(props: Props) {
+ super(props);
+ this.state = {
+ speaker_response: undefined,
+ speaker_slug: undefined,
+ };
+ }
+
+ componentDidMount() {
+ this.loadSpeakers();
+ }
+
+ public loadSpeakers() {
+ const url = this.props.apiConfig.getEndpoint('/voices');
+ fetch(url)
+ .then(res => res.json())
+ .then(
+ (result) => {
+ console.log('loaded speakers', result);
+ let first_speaker = result.speakers[0].slug;
+ this.setState({
+ speaker_response: result,
+ speaker_slug: first_speaker,
+ })
+ }
+ );
+ }
+
+ changeSpeaker = (ev: React.FormEvent) => {
+ const speakerSlug = (ev.target as HTMLInputElement).value;
+ this.setState({
+ speaker_slug: speakerSlug,
+ })
+ }
+
+ public render() {
+ let speakers = this.state.speaker_response?.speakers;
+ let dropdown;
+
+ if (speakers?.length || 0 > 0) {
+ dropdown =
+ }
+
+ return (
+
+ SpeakerSpectrogramMode
+ {dropdown}
+
+
+ );
+ }
+}
+
+export { SpeakerSpectrogramModeComponent };