Port old frontend (not quite working)

frontends/twitch-early-access/package-lock.json

@@ -1142,6 +1142,16 @@
         "minimist": "^1.2.0"
       }
     },
+    "@colormap/core": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/@colormap/core/-/core-0.2.0.tgz",
+      "integrity": "sha512-mFR7RHtSSDHwJq4pcXVa+MAzdRqADm5RiUkG35BfmUr9TZri/+NPfwJtDSvajzLwkOwBU0aGiSpNYX6qlOSwng=="
+    },
+    "@colormap/presets": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/@colormap/presets/-/presets-0.2.0.tgz",
+      "integrity": "sha512-hzUG6oS+dJxd26k6i4pxFDWP4InWhFRcptuv4U6Mq6/Vs7TT1aPZf1rWSfCv/FgJ11k/gKsUtkFdzVGHckBLxQ=="
+    },
     "@csstools/convert-colors": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@csstools/convert-colors/-/convert-colors-1.4.0.tgz",
@@ -2225,6 +2235,11 @@
         "@types/node": "*"
       }
     },
+    "@types/howler": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/@types/howler/-/howler-2.2.1.tgz",
+      "integrity": "sha512-1MiSldngr+eAO4lDPtjzl4Nf2GmRh8VDHIpNBIkyd25L22JExVlI6w3fjSM7+FNc1e1WZAPNq7/flkw685byfg=="
+    },
     "@types/html-minifier-terser": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz",
@@ -7126,6 +7141,11 @@
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz",
       "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg=="
     },
+    "howler": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/howler/-/howler-2.2.1.tgz",
+      "integrity": "sha512-0iIXvuBO/81CcrQ/HSSweYmbT50fT2mIc9XMFb+kxIfk2pW/iKzDbX1n3fZmDXMEIpYvyyfrB+gXwPYSDqUxIQ=="
+    },
     "hpack.js": {
       "version": "2.1.6",
       "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz",

frontends/twitch-early-access/package.json

@@ -3,13 +3,17 @@
   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "@colormap/core": "^0.2.0",
+    "@colormap/presets": "^0.2.0",
     "@testing-library/jest-dom": "^5.11.9",
     "@testing-library/react": "^11.2.5",
     "@testing-library/user-event": "^12.8.3",
+    "@types/howler": "^2.2.1",
     "@types/jest": "^26.0.20",
     "@types/node": "^12.20.6",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.2",
+    "howler": "^2.2.1",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
     "react-scripts": "4.0.3",

frontends/twitch-early-access/src/App.tsx

@@ -1,12 +1,40 @@
-import React from 'react';
 import './App.css';
 
-function App() {
-  return (
-    <div className="App">
-      Testing
-    </div>
-  );
+import React from 'react';
+import { ComponentFrame, ShowComponent } from './components/ComponentFrame';
+import ApiConfig from './api/ApiConfig';
+
+interface Props {
+}
+
+interface State {
+  showComponent: ShowComponent,
+  apiConfig: ApiConfig,
+  text: string,
+}
+
+class App extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      showComponent: ShowComponent.SPEAK,
+      apiConfig: new ApiConfig(),
+      text: "",
+    };
+  }
+
+  public updateText(text: string) {
+  }
+
+  public render() {
+    return <ComponentFrame 
+      showComponent={this.state.showComponent}
+      apiConfig={this.state.apiConfig}
+      text={this.state.text}
+      updateTextCallback={this.updateText}
+      />
+  }
 }
 
 export default App;

frontends/twitch-early-access/src/api/ApiConfig.ts

@@ -0,0 +1,70 @@
+
+enum Environment {
+  // Running in 'npm run-script start'
+  LOCALHOST_JS,
+  // Running under Rust server; 'npm run-script build'
+  LOCALHOST_RUST,
+  // Running on a deployed staging or production domain.
+  DEPLOYED,
+  // Running somewhere else
+  UNKNOWN,
+}
+
+class ApiConfig {
+
+  environment: Environment;
+
+  constructor() {
+    const url = new URL(window.location as any);
+
+    let environment;
+
+    switch (url.hostname) {
+      case "mumble.stream":
+      case "jungle.horse":
+        console.log('environment', 'deployed');
+        environment = Environment.DEPLOYED;
+        break;
+      case "localhost":
+        switch (url.port) {
+          case "8080":
+          case "7000":
+            console.log('environment', 'localhost_js');
+            environment = Environment.LOCALHOST_JS;
+            break;
+          default:
+            console.log('environment', 'localhost_rust');
+            environment = Environment.LOCALHOST_RUST;
+        }
+        break
+      default:
+        console.log('environment', 'unknown');
+        environment = Environment.UNKNOWN;
+    }
+
+    console.log('environment', environment);
+    
+    this.environment = environment;
+  }
+
+  public getEndpoint(path: string) : string {
+    let baseUrl;
+    switch (this.environment) {
+      case Environment.LOCALHOST_JS:
+        baseUrl = new URL('http://localhost:12345');
+        break;
+      case Environment.LOCALHOST_RUST:
+      case Environment.DEPLOYED:
+        // Directly access via the path.
+        return path;
+      case Environment.UNKNOWN:
+        baseUrl = new URL('http://mumble.stream');
+        break;
+    }
+
+    baseUrl.pathname = path;
+    return baseUrl.toString();
+  }
+}
+
+export default ApiConfig;

frontends/twitch-early-access/src/components/ComponentFrame.tsx

@@ -0,0 +1,50 @@
+import React from 'react';
+import ApiConfig from '../api/ApiConfig';
+import { SpeakerSpectrogramModeComponent } from './speaker_spectrogram_mode/SpeakerSpectrogramModeComponent';
+import { SpeakerModeComponent } from './speaker_mode/SpeakerModeComponent';
+
+enum ShowComponent {
+  SPEAK,
+  SPEAK_SPECTROGRAM,
+}
+
+interface Props {
+  showComponent: ShowComponent,
+  apiConfig: ApiConfig,
+  text: string,
+  updateTextCallback: (text: string) => void,
+}
+
+interface State {
+}
+
+class ComponentFrame extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+    };
+  }
+
+  public render() {
+
+    switch (this.props.showComponent) {
+      case ShowComponent.SPEAK_SPECTROGRAM:
+        return <SpeakerSpectrogramModeComponent 
+          apiConfig={this.props.apiConfig}
+          text={this.props.text}
+          updateTextCallback={this.props.updateTextCallback}
+          />;
+      case ShowComponent.SPEAK:
+      default:
+        return <SpeakerModeComponent 
+          apiConfig={this.props.apiConfig}
+          text={this.props.text}
+          updateTextCallback={this.props.updateTextCallback}
+          />;
+    }
+  }
+
+}
+
+export { ComponentFrame, ShowComponent }

frontends/twitch-early-access/src/components/speaker_mode/SpeakerAudioForm.tsx

@@ -0,0 +1,88 @@
+import React from 'react';
+import Howl from 'howler';
+import ApiConfig from '../../api/ApiConfig';
+import { SpeakRequest } from '../../api/ApiDefinition'
+
+interface Props {
+  apiConfig: ApiConfig,
+  speaker?: String,
+  text: string,
+  updateTextCallback: (text: string) => void,
+}
+
+interface State {
+  howl?: Howl.Howl,
+}
+
+
+class SpeakerAudioForm extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      howl: undefined,
+    };
+  }
+
+  componentDidMount() {
+    // OnMount events. An async CTOR, essentially.
+  }
+
+  handleTextChange = (ev: React.FormEvent<HTMLInputElement>) => {
+    const text = (ev.target as HTMLInputElement).value;
+    this.props.updateTextCallback(text);
+  }
+
+  makeRequest = (ev: React.FormEvent<HTMLFormElement>) => {
+    console.log("Form Submit");
+
+    if (!this.props.text) {
+      return;
+    }
+
+    let request = new SpeakRequest(this.props.text, this.props.speaker!);
+
+    const url = this.props.apiConfig.getEndpoint('/speak');
+
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(request),
+    })
+    .then(res => res.blob())
+    .then(blob => {
+      //console.log(blob);
+      const url = window.URL.createObjectURL(blob);
+
+      console.log('download at the audio url:');
+      console.log(url);
+
+      const sound = new Howl.Howl({
+        src: [url],
+        //format: 'wav',
+      });
+      
+      this.setState({howl: sound});
+      sound.play();
+
+      (window as any).sound = sound;
+    });
+
+    ev.preventDefault();
+    return false;
+  }
+
+  public render() {
+    return (
+      <form onSubmit={this.makeRequest}>
+        <input onChange={this.handleTextChange} value={this.props.text} />
+        <button>Submit</button>
+      </form>
+    );
+  }
+}
+
+export default SpeakerAudioForm;

frontends/twitch-early-access/src/components/speaker_mode/SpeakerDropdownComponent.tsx

@@ -0,0 +1,36 @@
+
+import React from 'react';
+import { SpeakerDetails } from './SpeakerResponse';
+
+interface Props {
+  changeCallback: (ev: React.FormEvent<HTMLSelectElement>) => void,
+  speakers: SpeakerDetails[]
+}
+
+interface State {
+}
+
+class SpeakerDropdownComponent extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+  }
+
+  public render() {
+    return (
+      <div>
+        <strong>Select Speaker</strong>
+        <br />
+        <select onChange={this.props.changeCallback}>
+          {this.props.speakers.map(function(speaker: SpeakerDetails) {
+            let name : string = speaker.name || "unknown";
+            let slug : string = speaker.slug || "unknown";
+            return (<option value={slug}>{name}</option>)
+          })};
+        </select>
+      </div>
+    );
+  }
+}
+
+export {SpeakerDropdownComponent};

frontends/twitch-early-access/src/components/speaker_mode/SpeakerModeComponent.tsx

@@ -0,0 +1,81 @@
+import React from 'react';
+import { SpeakerDetails, SpeakerResponse } from './SpeakerResponse';
+import { SpeakerDropdownComponent } from './SpeakerDropdownComponent';
+import SpeakerAudioForm from './SpeakerAudioForm';
+import ApiConfig from '../../api/ApiConfig';
+
+interface Props {
+  apiConfig: ApiConfig,
+  text: string,
+  updateTextCallback: (text: string) => void,
+}
+
+interface State {
+  speaker_response?: SpeakerResponse,
+  speaker_slug?: string,
+}
+
+class SpeakerModeComponent extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      speaker_response: undefined,
+      speaker_slug: undefined,
+    };
+  }
+
+  componentDidMount() {
+    this.loadSpeakers();
+  }
+
+  public loadSpeakers() {
+    const url = this.props.apiConfig.getEndpoint('/voices');
+    fetch(url)
+      .then(res => res.json())
+      .then(
+        (result) => {
+          console.log('loaded speakers', result);
+          let first_speaker = result.speakers[0].slug;
+          this.setState({
+            speaker_response: result,
+            speaker_slug: first_speaker,
+          })
+        }
+      );
+  }
+
+  changeSpeaker = (ev: React.FormEvent<HTMLSelectElement>) => {
+    const speakerSlug = (ev.target as HTMLInputElement).value;
+    this.setState({
+      speaker_slug: speakerSlug,
+    })
+  }
+
+  public render() {
+    let speakers = this.state.speaker_response?.speakers;
+    let dropdown;
+
+    if (speakers?.length || 0 > 0) {
+      dropdown = <SpeakerDropdownComponent 
+        speakers={speakers!} 
+        changeCallback={this.changeSpeaker}
+        />
+    }
+
+    return (
+      <div>
+        <b>SpeakerMode</b>
+        {dropdown}
+        <SpeakerAudioForm 
+          speaker={this.state.speaker_slug!} 
+          apiConfig={this.props.apiConfig}
+          text={this.props.text}
+          updateTextCallback={this.props.updateTextCallback}
+          />
+      </div>
+    );
+  }
+}
+
+export {SpeakerModeComponent};

frontends/twitch-early-access/src/components/speaker_mode/SpeakerResponse.ts

@@ -0,0 +1,15 @@
+
+
+class SpeakerResponse {
+  speakers?: SpeakerDetails[]
+}
+
+class SpeakerDetails {
+  name?: string
+  slug?: string
+  model_pipeline?: string
+  tacotron?: string
+  melgan?: string
+}
+
+export {SpeakerResponse, SpeakerDetails};

frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerDropdownComponent.tsx

@@ -0,0 +1,36 @@
+
+import React from 'react';
+import { SpeakerDetails } from './SpeakerResponse';
+
+interface Props {
+  changeCallback: (ev: React.FormEvent<HTMLSelectElement>) => void,
+  speakers: SpeakerDetails[]
+}
+
+interface State {
+}
+
+class SpeakerDropdownComponent extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+  }
+
+  public render() {
+    return (
+      <div>
+        <strong>Select Speaker</strong>
+        <br />
+        <select onChange={this.props.changeCallback}>
+          {this.props.speakers.map(function(speaker: SpeakerDetails) {
+            let name : string = speaker.name || "unknown";
+            let slug : string = speaker.slug || "unknown";
+            return (<option value={slug}>{name}</option>)
+          })};
+        </select>
+      </div>
+    );
+  }
+}
+
+export {SpeakerDropdownComponent};

frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerResponse.ts

@@ -0,0 +1,15 @@
+
+
+class SpeakerResponse {
+  speakers?: SpeakerDetails[]
+}
+
+class SpeakerDetails {
+  name?: string
+  slug?: string
+  model_pipeline?: string
+  tacotron?: string
+  melgan?: string
+}
+
+export {SpeakerResponse, SpeakerDetails};

frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramAudioForm.tsx

@@ -0,0 +1,175 @@
+import React from 'react';
+import Howl from 'howler';
+import ApiConfig from '../../api/ApiConfig';
+import { createColorMap, linearScale } from "@colormap/core";
+import { magma } from "@colormap/presets";
+import { SpeakRequest } from '../../api/ApiDefinition'
+
+interface Props {
+  apiConfig: ApiConfig,
+  speaker?: String,
+  text: string,
+  updateTextCallback: (text: string) => void,
+}
+
+interface State {
+  howl?: Howl.Howl,
+  spectrogram?: Spectrogram,
+}
+
+class Spectrogram {
+  pixels: Uint8ClampedArray;
+  width: number;
+  height: number;
+
+  constructor(pixels: Uint8ClampedArray, width: number, height: number) {
+    this.pixels = pixels;
+    this.width = width;
+    this.height = height;
+  }
+}
+
+class SpeakerSpectrogramAudioForm extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      howl: undefined,
+      spectrogram: undefined,
+    };
+  }
+
+  handleTextChange = (ev: React.FormEvent<HTMLInputElement>) => {
+    const text = (ev.target as HTMLInputElement).value;
+    this.props.updateTextCallback(text);
+  }
+
+  makeRequest = (ev: React.FormEvent<HTMLFormElement>) => {
+    console.log("Form Submit");
+
+    if (!this.props.text) {
+      return;
+    }
+
+    let request = new SpeakRequest(this.props.text, this.props.speaker!);
+
+    const url = this.props.apiConfig.getEndpoint('/speak_spectrogram');
+
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(request),
+    })
+    .then(res => res.json())
+    .then(res => {
+      const data = `data:audio/wav;base64,${res.audio_base64}`;
+      
+      const audioUrl = new URL(data);
+
+      // var image = new Image();
+      // image.src = `data:image/bmp;base64,${res.spectrogram.bytes_base64}`;
+      // console.log('image', image);
+
+      let scale = linearScale([0, 255], [0, 1]);
+      let colorMap = createColorMap(magma, scale);
+
+      // https://stackoverflow.com/a/21797381
+      function base64ToArrayBuffer(base64string: string) : Uint8ClampedArray {
+        var binary_string = window.atob(base64string);
+        var len = binary_string.length * 4;
+        let bytes = new Uint8ClampedArray(len);
+        for (let i = 0, j = 0; i < len; i++, j += 4) {
+          let val = binary_string.charCodeAt(i);
+          if (isNaN(val)) {
+            val = 0;
+          }
+          let mapped = colorMap(val);
+          let r = Math.floor(mapped[0] * 255)
+          let g = Math.floor(mapped[1] * 255)
+          let b = Math.floor(mapped[2] * 255)
+          bytes[j+0] = r;
+          bytes[j+1] = g;
+          bytes[j+2] = b;
+          bytes[j+3] = 255;
+        }
+        return bytes;
+      }
+
+      let bytes = base64ToArrayBuffer(res.spectrogram.bytes_base64);
+
+      const spectrogram = new Spectrogram(bytes, res.spectrogram.width, res.spectrogram.height);
+
+      const sound = new Howl.Howl({
+        src: [data],
+        //format: 'wav',
+      });
+      
+      this.setState({
+        howl: sound,
+        spectrogram: spectrogram,
+      });
+
+      sound.play();
+
+      (window as any).sound = sound;
+    });
+
+    ev.preventDefault();
+    return false;
+
+  }
+
+  componentDidMount() {
+    this.updateCanvas();
+  }
+
+  componentDidUpdate() {
+    this.updateCanvas();
+  }
+
+  updateCanvas() {
+    const ctx = (this.refs.canvas as any).getContext('2d');
+    // let width = 300;
+    // let height = 80;
+    // ctx.clearRect(0,0, width, height);
+
+    if (this.state.spectrogram !== undefined) {
+      let width = this.state.spectrogram!.width;
+      let height = this.state.spectrogram!.height;
+
+      var image = new ImageData(this.state.spectrogram!.pixels, width, height);
+
+      createImageBitmap(image).then(renderer => {
+        ctx.drawImage(renderer, 0, 0, width * 3, height * 3)
+      });
+    }
+  }
+
+  public render() {
+    let width = 150 * 3;
+    let height = 80 * 3;
+
+    if (this.state.spectrogram !== undefined) {
+      width = this.state.spectrogram.width * 3;
+      height = this.state.spectrogram.height * 3;
+    }
+
+    let canvas = <canvas ref="canvas" width={width} height={height} id="spectrogram" />
+
+    return (
+      <div>
+        {canvas}
+
+        <form onSubmit={this.makeRequest}>
+          <input onChange={this.handleTextChange} value={this.props.text} />
+          <button>Submit</button>
+        </form>
+      </div>
+    );
+  }
+}
+
+export default SpeakerSpectrogramAudioForm;

frontends/twitch-early-access/src/components/speaker_spectrogram_mode/SpeakerSpectrogramModeComponent.tsx

@@ -0,0 +1,81 @@
+import React from 'react';
+import { SpeakerDetails, SpeakerResponse } from './SpeakerResponse';
+import { SpeakerDropdownComponent } from './SpeakerDropdownComponent';
+import SpeakerSpectrogramAudioForm from './SpeakerSpectrogramAudioForm';
+import ApiConfig from '../../api/ApiConfig';
+
+interface Props {
+  apiConfig: ApiConfig,
+  text: string,
+  updateTextCallback: (text: string) => void,
+}
+
+interface State {
+  speaker_response?: SpeakerResponse,
+  speaker_slug?: string,
+}
+
+class SpeakerSpectrogramModeComponent extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      speaker_response: undefined,
+      speaker_slug: undefined,
+    };
+  }
+
+  componentDidMount() {
+    this.loadSpeakers();
+  }
+
+  public loadSpeakers() {
+    const url = this.props.apiConfig.getEndpoint('/voices');
+    fetch(url)
+      .then(res => res.json())
+      .then(
+        (result) => {
+          console.log('loaded speakers', result);
+          let first_speaker = result.speakers[0].slug;
+          this.setState({
+            speaker_response: result,
+            speaker_slug: first_speaker,
+          })
+        }
+      );
+  }
+
+  changeSpeaker = (ev: React.FormEvent<HTMLSelectElement>) => {
+    const speakerSlug = (ev.target as HTMLInputElement).value;
+    this.setState({
+      speaker_slug: speakerSlug,
+    })
+  }
+
+  public render() {
+    let speakers = this.state.speaker_response?.speakers;
+    let dropdown;
+
+    if (speakers?.length || 0 > 0) {
+      dropdown = <SpeakerDropdownComponent 
+        speakers={speakers!} 
+        changeCallback={this.changeSpeaker}
+        />
+    }
+
+    return (
+      <div>
+        <b>SpeakerSpectrogramMode</b>
+        {dropdown}
+        <SpeakerSpectrogramAudioForm 
+          speaker={this.state.speaker_slug!} 
+          apiConfig={this.props.apiConfig}
+          text={this.props.text}
+          updateTextCallback={this.props.updateTextCallback}
+          />
+      </div>
+    );
+  }
+}
+
+export { SpeakerSpectrogramModeComponent };