hacky spectrograms!

tts_frontend/src/components/ModalComponent.tsx

@@ -3,9 +3,11 @@ import React from 'react';
 import { ModeSelector } from './ModeSelector';
 import { SentencesComponent } from './modes/sentences/SentencesComponent'
 import { MainComponent } from './MainComponent'
+import { SpeakerSpectrogramModeComponent } from './modes/speaker_spectrogram_mode/SpeakerSpectrogramModeComponent';
 
 enum Mode {
   SPEAKER,
+  SPEAKER_SPECTROGRAM,
   ADVANCED,
   SENTENCE,
 }
@@ -41,6 +43,9 @@ class ModalComponent extends React.Component<Props, State> {
         // Share the text input, but change the form logic and dropdowns.
         component = <MainComponent apiConfig={this.props.apiConfig} mode={this.state.mode } />;
         break;
+      case Mode.SPEAKER_SPECTROGRAM:
+        component = <SpeakerSpectrogramModeComponent apiConfig={this.props.apiConfig} />;
+        break;
       case Mode.SENTENCE:
         component = <SentencesComponent apiConfig={this.props.apiConfig} />;
         break;

tts_frontend/src/components/ModeSelector.tsx

@@ -17,6 +17,7 @@ class ModeSelector extends React.Component<Props, State> {
 
   public render() {
     let speakerDisabled = false;
+    let speakerSpectrogramDisabled = false;
     let advancedDisabled = false;
     let sentencesDisabled = false;
 
@@ -24,6 +25,9 @@ class ModeSelector extends React.Component<Props, State> {
       case Mode.SPEAKER:
         speakerDisabled = true;
         break;
+      case Mode.SPEAKER_SPECTROGRAM:
+        speakerSpectrogramDisabled = true;
+        break;
       case Mode.ADVANCED:
         advancedDisabled = true;
         break;
@@ -39,6 +43,11 @@ class ModeSelector extends React.Component<Props, State> {
           onClick={() => this.props.switchModeCallback(Mode.SPEAKER)}>
             Speakers
         </button>
+        <button 
+          disabled={speakerSpectrogramDisabled} 
+          onClick={() => this.props.switchModeCallback(Mode.SPEAKER_SPECTROGRAM)}>
+            Speaker + Spectrogram
+        </button>
         <button 
           disabled={advancedDisabled}
           onClick={() => this.props.switchModeCallback(Mode.ADVANCED)}>

tts_frontend/src/components/modes/speaker_spectrogram_mode/SpeakerDropdownComponent.tsx

@@ -0,0 +1,36 @@
+
+import React from 'react';
+import { SpeakerDetails } from './SpeakerResponse';
+
+interface Props {
+  changeCallback: (ev: React.FormEvent<HTMLSelectElement>) => void,
+  speakers: SpeakerDetails[]
+}
+
+interface State {
+}
+
+class SpeakerDropdownComponent extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+  }
+
+  public render() {
+    return (
+      <div>
+        <strong>Select Speaker</strong>
+        <br />
+        <select onChange={this.props.changeCallback}>
+          {this.props.speakers.map(function(speaker: SpeakerDetails) {
+            let name : string = speaker.name || "unknown";
+            let slug : string = speaker.slug || "unknown";
+            return (<option value={slug}>{name}</option>)
+          })};
+        </select>
+      </div>
+    );
+  }
+}
+
+export {SpeakerDropdownComponent};

tts_frontend/src/components/modes/speaker_spectrogram_mode/SpeakerResponse.ts

@@ -0,0 +1,15 @@
+
+
+class SpeakerResponse {
+  speakers?: SpeakerDetails[]
+}
+
+class SpeakerDetails {
+  name?: string
+  slug?: string
+  model_pipeline?: string
+  tacotron?: string
+  melgan?: string
+}
+
+export {SpeakerResponse, SpeakerDetails};

tts_frontend/src/components/modes/speaker_spectrogram_mode/SpeakerSpectrogramAudioForm.tsx

@@ -0,0 +1,174 @@
+import React from 'react';
+import Howl from 'howler';
+import ApiConfig from '../../../ApiConfig';
+import { SpeakRequest } from '../../../api/ApiDefinition'
+
+interface Props {
+  apiConfig: ApiConfig,
+  speaker?: String,
+}
+
+interface State {
+  text?: String,
+  howl?: Howl,
+  spectrogram?: Spectrogram,
+}
+
+class Spectrogram {
+  pixels: Uint8Array;
+  width: number;
+  height: number;
+
+  constructor(pixels: Uint8Array, width: number, height: number) {
+    this.pixels = pixels;
+    this.width = width;
+    this.height = height;
+  }
+}
+
+class SpeakerSpectrogramAudioForm extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      text: undefined,
+      howl: undefined,
+      spectrogram: undefined,
+    };
+  }
+
+  handleTextChange = (ev: React.FormEvent<HTMLInputElement>) => {
+    const text = (ev.target as HTMLInputElement).value;
+    this.setState({text: text});
+  }
+
+  makeRequest = (ev: React.FormEvent<HTMLFormElement>) => {
+    console.log("Form Submit");
+
+    if (!this.state.text) {
+      return;
+    }
+
+    let request = new SpeakRequest(this.state.text, this.props.speaker!);
+
+    const url = this.props.apiConfig.getEndpoint('/speak_spectrogram');
+
+    fetch(url, {
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(request),
+    })
+    .then(res => res.json())
+    .then(res => {
+      const data = `data:audio/wav;base64,${res.audio_base64}`;
+
+      // var image = new Image();
+      // image.src = `data:image/bmp;base64,${res.spectrogram.bytes_base64}`;
+      // console.log('image', image);
+
+      // https://stackoverflow.com/a/21797381
+      function base64ToArrayBuffer(base64string: string) : Uint8Array {
+        var binary_string = window.atob(base64string);
+        var len = binary_string.length;
+        var bytes = new Uint8Array(len);
+        for (var i = 0; i < len; i++) {
+            bytes[i] = binary_string.charCodeAt(i);
+        }
+        //return bytes.buffer;
+        return bytes;
+      }
+
+      let bytes = base64ToArrayBuffer(res.spectrogram.bytes_base64);
+
+      const spectrogram = new Spectrogram(bytes, res.spectrogram.width, res.spectrogram.height);
+
+      const sound = new Howl.Howl({
+        src: [data],
+        format: 'wav',
+      });
+      
+      this.setState({
+        howl: sound,
+        spectrogram: spectrogram,
+      });
+
+      sound.play();
+
+      (window as any).sound = sound;
+    });
+
+    ev.preventDefault();
+    return false;
+
+  }
+
+  componentDidMount() {
+    this.updateCanvas();
+  }
+
+  componentDidUpdate() {
+    this.updateCanvas();
+  }
+
+  updateCanvas() {
+    const ctx = (this.refs.canvas as any).getContext('2d');
+
+    let width = 300;
+    let height = 80;
+    ctx.clearRect(0,0, width, height);
+
+    if (this.state.spectrogram !== undefined) {
+      width = this.state.spectrogram!.width || 300;
+      height = this.state.spectrogram!.height || 80;
+
+      ctx.clearRect(0,0, width, height);
+
+      //for (let i = 0; i < width; i++) {
+      //  for (let j = 0; j < height; j++) {
+      //  }
+      //}
+
+      var imageData = ctx.getImageData(0, 0, width, height);
+      var data = imageData.data;
+
+      for (let i = 0, j = 0; i < data.length; i += 4, j += 1) {
+        let value = this.state.spectrogram!.pixels[j];
+
+        data[i]     = value; // red
+        data[i + 1] = value; // green
+        data[i + 2] = value; // blue
+        data[i + 3] = 255;
+      }
+      console.log('updating context');
+      ctx.putImageData(imageData, 0, 0);
+    }
+  }
+
+  public render() {
+    let width = 150;
+    let height = 80;
+    if (this.state.spectrogram !== undefined) {
+      width = this.state.spectrogram.width;
+      height = this.state.spectrogram.height;
+
+    }
+
+    let canvas = <canvas ref="canvas" width={width} height={height}/>
+
+    return (
+      <div>
+        {canvas}
+
+        <form onSubmit={this.makeRequest}>
+          <input onChange={this.handleTextChange} />
+          <button>Submit</button>
+        </form>
+      </div>
+    );
+  }
+}
+
+export default SpeakerSpectrogramAudioForm;

tts_frontend/src/components/modes/speaker_spectrogram_mode/SpeakerSpectrogramModeComponent.tsx

@@ -0,0 +1,77 @@
+import React from 'react';
+import { SpeakerDetails, SpeakerResponse } from './SpeakerResponse';
+import { SpeakerDropdownComponent } from './SpeakerDropdownComponent';
+import SpeakerSpectrogramAudioForm from './SpeakerSpectrogramAudioForm';
+import ApiConfig from '../../../ApiConfig';
+
+interface Props {
+  apiConfig: ApiConfig,
+}
+
+interface State {
+  speaker_response?: SpeakerResponse,
+  speaker_slug?: string,
+}
+
+class SpeakerSpectrogramModeComponent extends React.Component<Props, State> {
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      speaker_response: undefined,
+      speaker_slug: undefined,
+    };
+  }
+
+  componentDidMount() {
+    this.loadSpeakers();
+  }
+
+  public loadSpeakers() {
+    const url = this.props.apiConfig.getEndpoint('/speakers');
+    fetch(url)
+      .then(res => res.json())
+      .then(
+        (result) => {
+          console.log('loaded speakers', result);
+          let first_speaker = result.speakers[0].slug;
+          this.setState({
+            speaker_response: result,
+            speaker_slug: first_speaker,
+          })
+        }
+      );
+  }
+
+  changeSpeaker = (ev: React.FormEvent<HTMLSelectElement>) => {
+    const speakerSlug = (ev.target as HTMLInputElement).value;
+    this.setState({
+      speaker_slug: speakerSlug,
+    })
+  }
+
+  public render() {
+    let speakers = this.state.speaker_response?.speakers;
+    let dropdown;
+
+    if (speakers?.length || 0 > 0) {
+      dropdown = <SpeakerDropdownComponent 
+        speakers={speakers!} 
+        changeCallback={this.changeSpeaker}
+        />
+    }
+
+    return (
+      <div>
+        <b>SpeakerSpectrogramMode</b>
+        {dropdown}
+        <SpeakerSpectrogramAudioForm 
+          speaker={this.state.speaker_slug!} 
+          apiConfig={this.props.apiConfig}
+          />
+      </div>
+    );
+  }
+}
+
+export { SpeakerSpectrogramModeComponent };

tts_service/models.toml

@@ -30,7 +30,8 @@ name = "John F. Kennedy"
 slug = "jfk"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" # TODO: Wtf - noise
 speaker_id = 1
 
 [[speakers]]
@@ -38,7 +39,8 @@ name = "Arnold Schwarzenegger"
 slug = "arnold"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 2
 
 [[speakers]]
@@ -46,7 +48,8 @@ name = "Bill Gates"
 slug = "gates"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 3
 
 [[speakers]]
@@ -54,7 +57,8 @@ name = "Mark Zuckerberg"
 slug = "zuckerberg"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 4
 
 [[speakers]]
@@ -62,7 +66,8 @@ name = "Ronald Reagan"
 slug = "reagan"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 5
 
 [[speakers]]
@@ -70,7 +75,8 @@ name = "Queen Elizabeth II"
 slug = "queen-elizabeth"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 6
 
 [[speakers]]
@@ -78,7 +84,8 @@ name = "Mario"
 slug = "mario"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 7
 
 [[speakers]]
@@ -86,7 +93,8 @@ name = "Zapp Brannigan"
 slug = "zapp"
 model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
-melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 8
 
 [[speakers]]
@@ -96,9 +104,35 @@ model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
 glow_tts_multi_speaker = "glow_multispeaker.torchjit"
 #melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
 #melgan = "shared_melgan_container-2020.01.05.pt"
-melgan = "melgan_ljs-high-pitch_container.pt"
+#melgan = "melgan_ljs-high-pitch_container.pt"
+melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 speaker_id = 9
 
+[[speakers]]
+name = "LJS Multi Base 1"
+slug = "multi1"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 1
+
+[[speakers]]
+name = "LJS Multi Base 2"
+slug = "multi1"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 2
+
+[[speakers]]
+name = "LJS Multi Base 50"
+slug = "multi1"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 50
+
+
 # --------------- Model configs ---------------
 
 [[model_locations]]
@@ -244,6 +278,10 @@ description = "First experiments (continued); modified with longer input to JIT"
 file_path = "glow_multispeaker.torchjit"
 description = "Mutli-speaker garbage"
 
+[[glow_tts_multi_speaker]]
+file_path = "glow_ljs_base.torchjit"
+description = "Mutli-speaker base model (LambdaLabs expensive Tesla GPU)"
+
 # ---------------- Melgan ----------------
 
 # TODO: Get a high-quality LJS Melgan
@@ -272,3 +310,18 @@ description = "Unknown"
 file_path = "shared_melgan_container-2020.01.05.pt"
 description = "Unknown"
 
+[[melgan]]
+file_path = "melgan_multivoice-1.0_2020.06.15_12d5838_6275.torchjit"
+description = "first multivoice (txlearned)"
+
+[[melgan]]
+file_path = "melgan_multivoice-1.0_2020.06.15_12d5838_6300.torchjit"
+description = "first multivoice (txlearned)"
+
+[[melgan]]
+file_path = "melgan_multivoice-1.0_2020.06.15_12d5838_6475.torchjit"
+description = "first multivoice (txlearned)"
+
+[[melgan]]
+file_path = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
+description = "first multivoice (txlearned)"