Add my demo_server.py implementation from CycleGAN

cycle_gan/demo_server.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+
+"""
+Starts a demo HTTP server to capture and transform audio
+as a live demonstration of the trained model.
+
+Brandon Thomas 2019-07-29 <bt@brand.io> <echelon@gmail.com>
+"""
+
+import argparse
+import falcon
+import io
+import librosa
+import numpy as np
+import os
+import scipy
+import soundfile
+import tensorflow as tf
+import subprocess
+import tempfile
+
+from falcon_multipart.middleware import MultipartMiddleware
+from model import CycleGAN
+from preprocess import *
+from wsgiref import simple_server
+
+print("TensorFlow version: {}".format(tf.version.VERSION))
+
+INDEX_HTML = '''
+<!doctype html>
+<html>
+  <body>
+    <h1>Voice Demo</h1>
+    <script src="./script/recorder.js" type="application/javascript"></script>
+    <script type="application/javascript">
+      var audio_context;
+      var recorder;
+
+      // NB: Capture code originates from 'recorder.js' demo.
+      function startUserMedia(stream) {
+        var input = audio_context.createMediaStreamSource(stream);
+        window.recorder = new Recorder(input);
+      }
+
+      function createDownloadLink() {
+        window.recorder && window.recorder.exportWAV(function(blob) {
+          var xhr = new XMLHttpRequest();
+          xhr.responseType = 'blob';
+          xhr.onload=function(e) {
+            console.log('response received');
+            var objectUrl = window.URL.createObjectURL(this.response);
+            console.log('audio URL', objectUrl);
+            var audio = document.getElementById('audio');
+            audio.src = objectUrl;
+            audio.play();
+          };
+
+          var fd = new FormData();
+          fd.append("audio_data", blob, "audio_file.wav");
+          console.log(blob);
+          console.log('sending...');
+          xhr.open("POST","/upload",true);
+          xhr.send(fd);
+          console.log('sent');
+        });
+      }
+
+      function startRecording(button) {
+        window.recorder && window.recorder.record();
+        document.getElementById('start').disabled = true;
+        document.getElementById('stop').disabled = false;
+      }
+
+      function stopRecording(button) {
+        window.recorder && window.recorder.stop();
+        document.getElementById('start').disabled = false;
+        document.getElementById('stop').disabled = true;
+
+        createDownloadLink();
+        window.recorder.clear();
+      }
+
+      window.onload = function init() {
+        try {
+          audio_context = new AudioContext();
+        } catch (e) {
+          alert('No web audio support in this browser!');
+        }
+
+        var media = navigator.mediaDevices.getUserMedia({ audio: true, video: false });
+        window.media = media;
+        console.log('media', media);
+        media.then(startUserMedia);
+      };
+    </script>
+
+    <button id="start" onclick="startRecording(this);">record</button>
+    <br>
+    <br>
+    <button id="stop" onclick="stopRecording(this);" disabled>stop</button>
+    <br>
+    <br>
+    <audio controls id="audio">
+      <source id="source" src="" type="audio/wav">
+      Your browser does not support the audio element.
+    </audio>
+  </body>
+</html>
+'''
+
+
+class Converter():
+    def __init__(self, model_dir, model_name):
+        self.num_features = 24
+        self.sampling_rate = 16000
+        self.frame_period = 5.0
+
+        self.model = CycleGAN(num_features = self.num_features, mode = 'test')
+
+        self.model.load(filepath = os.path.join(model_dir, model_name))
+
+        """
+        # NB: Save the graph
+        definition = self.model.sess.graph_def
+        directory = 'saved_model_2'
+        tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True)
+
+        # https://github.com/tensorflow/models/issues/3530#issuecomment-395968881
+        output_dir = './saved_model/'
+        builder = tf.saved_model.builder.SavedModelBuilder(output_dir)
+
+        builder.add_meta_graph_and_variables(
+            self.model.sess,
+            [tf.saved_model.tag_constants.SERVING],
+            main_op=tf.tables_initializer(),
+        )
+
+        builder.save()
+        """
+
+        """
+        builder.add_meta_graph_and_variables(
+            self.model.sess,
+            [tf.saved_model.tag_constants.SERVING],
+            signature_def_map={
+                'predict_images':
+                    prediction_signature,
+                signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+                    classification_signature,
+            },
+            main_op=tf.tables_initializer())
+        """
+
+        self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz'))
+        self.mcep_mean_A = self.mcep_normalization_params['mean_A']
+        self.mcep_std_A = self.mcep_normalization_params['std_A']
+        self.mcep_mean_B = self.mcep_normalization_params['mean_B']
+        self.mcep_std_B = self.mcep_normalization_params['std_B']
+
+        self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz'))
+        self.logf0s_mean_A = self.logf0s_normalization_params['mean_A']
+        self.logf0s_std_A = self.logf0s_normalization_params['std_A']
+        self.logf0s_mean_B = self.logf0s_normalization_params['mean_B']
+        self.logf0s_std_B = self.logf0s_normalization_params['std_B']
+
+    def convert(self, wav, conversion_direction='A2B'):
+        wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4)
+        f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period)
+        coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features)
+        coded_sp_transposed = coded_sp.T
+
+        if conversion_direction == 'A2B':
+            f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B)
+            coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A
+            coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0]
+            coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B
+        else:
+            f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A)
+            coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B
+            coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0]
+            coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A
+
+        coded_sp_converted = coded_sp_converted.T
+        coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
+        decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate)
+        wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period)
+
+        # For debugging model output, uncomment the following line:
+        # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate)
+
+        # TODO: Perhaps ditch this. It's probably unnecessary work.
+        upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000)
+        pcm_data = upsampled.astype(np.float64)
+        stereo_pcm_data = np.tile(pcm_data, (2,1)).T
+
+        buf = io.BytesIO()
+        scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32))
+        return buf
+
+# Set up model
+# This should live long in memory, so we do it up front.
+model_dir_default = './model/sf1_tm1'
+model_name_default = 'sf1_tm1.ckpt'
+converter = Converter(model_dir_default, model_name_default)
+
+class IndexHandler():
+    def on_get(self, request, response):
+        response.content_type = 'text/html'
+        response.body = INDEX_HTML
+
+class ApiHandler():
+    def on_post(self, request, response):
+        # NB: uses middleware to pull out data.
+        form_data = request.params['audio_data'].file
+        data, samplerate = soundfile.read(form_data)
+
+        # For debugging browser input, uncomment the following line:
+        # scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data)
+
+        # NB: Convert the input stereo signal into mono.
+        # In the future the frontend should be responsible for sampling details.
+        mono = data[:, 0]
+
+        # NB: We must downsample to the rate that the network is trained on.
+        downsampled = librosa.resample(mono, samplerate, 16000)
+
+        # Evaluate the model
+        print(">>> Converting...")
+        results = converter.convert(downsampled, conversion_direction = 'A2B')
+
+        temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio')
+        temp_file = tempfile.NamedTemporaryFile(suffix='.wav')
+
+        temp_file.write(results.read())
+
+        out_file = temp_dir.name + '/output.ogg'
+
+        # NB: Browsers have a great deal of trouble decoding WAV files unless they are in the
+        # narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python
+        # tools do a good job of this, so here we shell out to ffmpeg and generate OGG.
+        # It's lazy and messy, but it works for now.
+        # See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape
+        # See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs
+        _stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file])
+
+        response.content_type = 'audio/ogg'
+        with open(out_file, mode='rb') as f:
+            response.data = f.read()
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--port', type=int, default=8000)
+    args = parser.parse_args()
+
+    api = falcon.API(middleware=[MultipartMiddleware()])
+    api.add_route('/', IndexHandler())
+    api.add_route('/upload', ApiHandler())
+    api.add_static_route('/script', os.path.abspath('./script'))
+    api.add_static_route('/sound', os.path.abspath('./sound'))
+    print('Serving on 0.0.0.0:%d' % args.port)
+    simple_server.make_server('0.0.0.0', args.port, api).serve_forever()
+
+if __name__ == '__main__':
+    main()
+

cycle_gan/script/recorder.js

@@ -0,0 +1,357 @@
+(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Recorder = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
+"use strict";
+
+module.exports = require("./recorder").Recorder;
+
+},{"./recorder":2}],2:[function(require,module,exports){
+'use strict';
+
+var _createClass = (function () {
+    function defineProperties(target, props) {
+        for (var i = 0; i < props.length; i++) {
+            var descriptor = props[i];descriptor.enumerable = descriptor.enumerable || false;descriptor.configurable = true;if ("value" in descriptor) descriptor.writable = true;Object.defineProperty(target, descriptor.key, descriptor);
+        }
+    }return function (Constructor, protoProps, staticProps) {
+        if (protoProps) defineProperties(Constructor.prototype, protoProps);if (staticProps) defineProperties(Constructor, staticProps);return Constructor;
+    };
+})();
+
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+exports.Recorder = undefined;
+
+var _inlineWorker = require('inline-worker');
+
+var _inlineWorker2 = _interopRequireDefault(_inlineWorker);
+
+function _interopRequireDefault(obj) {
+    return obj && obj.__esModule ? obj : { default: obj };
+}
+
+function _classCallCheck(instance, Constructor) {
+    if (!(instance instanceof Constructor)) {
+        throw new TypeError("Cannot call a class as a function");
+    }
+}
+
+var Recorder = exports.Recorder = (function () {
+    function Recorder(source, cfg) {
+        var _this = this;
+
+        _classCallCheck(this, Recorder);
+
+        this.config = {
+            bufferLen: 4096,
+            numChannels: 2,
+            mimeType: 'audio/wav'
+        };
+        this.recording = false;
+        this.callbacks = {
+            getBuffer: [],
+            exportWAV: []
+        };
+
+        Object.assign(this.config, cfg);
+        this.context = source.context;
+        this.node = (this.context.createScriptProcessor || this.context.createJavaScriptNode).call(this.context, this.config.bufferLen, this.config.numChannels, this.config.numChannels);
+
+        this.node.onaudioprocess = function (e) {
+            if (!_this.recording) return;
+
+            var buffer = [];
+            for (var channel = 0; channel < _this.config.numChannels; channel++) {
+                buffer.push(e.inputBuffer.getChannelData(channel));
+            }
+            _this.worker.postMessage({
+                command: 'record',
+                buffer: buffer
+            });
+        };
+
+        source.connect(this.node);
+        this.node.connect(this.context.destination); //this should not be necessary
+
+        var self = {};
+        this.worker = new _inlineWorker2.default(function () {
+            var recLength = 0,
+                recBuffers = [],
+                sampleRate = undefined,
+                numChannels = undefined;
+
+            self.onmessage = function (e) {
+                switch (e.data.command) {
+                    case 'init':
+                        init(e.data.config);
+                        break;
+                    case 'record':
+                        record(e.data.buffer);
+                        break;
+                    case 'exportWAV':
+                        exportWAV(e.data.type);
+                        break;
+                    case 'getBuffer':
+                        getBuffer();
+                        break;
+                    case 'clear':
+                        clear();
+                        break;
+                }
+            };
+
+            function init(config) {
+                sampleRate = config.sampleRate;
+                numChannels = config.numChannels;
+                initBuffers();
+            }
+
+            function record(inputBuffer) {
+                for (var channel = 0; channel < numChannels; channel++) {
+                    recBuffers[channel].push(inputBuffer[channel]);
+                }
+                recLength += inputBuffer[0].length;
+            }
+
+            function exportWAV(type) {
+                var buffers = [];
+                for (var channel = 0; channel < numChannels; channel++) {
+                    buffers.push(mergeBuffers(recBuffers[channel], recLength));
+                }
+                var interleaved = undefined;
+                if (numChannels === 2) {
+                    interleaved = interleave(buffers[0], buffers[1]);
+                } else {
+                    interleaved = buffers[0];
+                }
+                var dataview = encodeWAV(interleaved);
+                var audioBlob = new Blob([dataview], { type: type });
+
+                self.postMessage({ command: 'exportWAV', data: audioBlob });
+            }
+
+            function getBuffer() {
+                var buffers = [];
+                for (var channel = 0; channel < numChannels; channel++) {
+                    buffers.push(mergeBuffers(recBuffers[channel], recLength));
+                }
+                self.postMessage({ command: 'getBuffer', data: buffers });
+            }
+
+            function clear() {
+                recLength = 0;
+                recBuffers = [];
+                initBuffers();
+            }
+
+            function initBuffers() {
+                for (var channel = 0; channel < numChannels; channel++) {
+                    recBuffers[channel] = [];
+                }
+            }
+
+            function mergeBuffers(recBuffers, recLength) {
+                var result = new Float32Array(recLength);
+                var offset = 0;
+                for (var i = 0; i < recBuffers.length; i++) {
+                    result.set(recBuffers[i], offset);
+                    offset += recBuffers[i].length;
+                }
+                return result;
+            }
+
+            function interleave(inputL, inputR) {
+                var length = inputL.length + inputR.length;
+                var result = new Float32Array(length);
+
+                var index = 0,
+                    inputIndex = 0;
+
+                while (index < length) {
+                    result[index++] = inputL[inputIndex];
+                    result[index++] = inputR[inputIndex];
+                    inputIndex++;
+                }
+                return result;
+            }
+
+            function floatTo16BitPCM(output, offset, input) {
+                for (var i = 0; i < input.length; i++, offset += 2) {
+                    var s = Math.max(-1, Math.min(1, input[i]));
+                    output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+                }
+            }
+
+            function writeString(view, offset, string) {
+                for (var i = 0; i < string.length; i++) {
+                    view.setUint8(offset + i, string.charCodeAt(i));
+                }
+            }
+
+            function encodeWAV(samples) {
+                var buffer = new ArrayBuffer(44 + samples.length * 2);
+                var view = new DataView(buffer);
+
+                /* RIFF identifier */
+                writeString(view, 0, 'RIFF');
+                /* RIFF chunk length */
+                view.setUint32(4, 36 + samples.length * 2, true);
+                /* RIFF type */
+                writeString(view, 8, 'WAVE');
+                /* format chunk identifier */
+                writeString(view, 12, 'fmt ');
+                /* format chunk length */
+                view.setUint32(16, 16, true);
+                /* sample format (raw) */
+                view.setUint16(20, 1, true);
+                /* channel count */
+                view.setUint16(22, numChannels, true);
+                /* sample rate */
+                view.setUint32(24, sampleRate, true);
+                /* byte rate (sample rate * block align) */
+                view.setUint32(28, sampleRate * 4, true);
+                /* block align (channel count * bytes per sample) */
+                view.setUint16(32, numChannels * 2, true);
+                /* bits per sample */
+                view.setUint16(34, 16, true);
+                /* data chunk identifier */
+                writeString(view, 36, 'data');
+                /* data chunk length */
+                view.setUint32(40, samples.length * 2, true);
+
+                floatTo16BitPCM(view, 44, samples);
+
+                return view;
+            }
+        }, self);
+
+        this.worker.postMessage({
+            command: 'init',
+            config: {
+                sampleRate: this.context.sampleRate,
+                numChannels: this.config.numChannels
+            }
+        });
+
+        this.worker.onmessage = function (e) {
+            var cb = _this.callbacks[e.data.command].pop();
+            if (typeof cb == 'function') {
+                cb(e.data.data);
+            }
+        };
+    }
+
+    _createClass(Recorder, [{
+        key: 'record',
+        value: function record() {
+            this.recording = true;
+        }
+    }, {
+        key: 'stop',
+        value: function stop() {
+            this.recording = false;
+        }
+    }, {
+        key: 'clear',
+        value: function clear() {
+            this.worker.postMessage({ command: 'clear' });
+        }
+    }, {
+        key: 'getBuffer',
+        value: function getBuffer(cb) {
+            cb = cb || this.config.callback;
+            if (!cb) throw new Error('Callback not set');
+
+            this.callbacks.getBuffer.push(cb);
+
+            this.worker.postMessage({ command: 'getBuffer' });
+        }
+    }, {
+        key: 'exportWAV',
+        value: function exportWAV(cb, mimeType) {
+            mimeType = mimeType || this.config.mimeType;
+            cb = cb || this.config.callback;
+            if (!cb) throw new Error('Callback not set');
+
+            this.callbacks.exportWAV.push(cb);
+
+            this.worker.postMessage({
+                command: 'exportWAV',
+                type: mimeType
+            });
+        }
+    }], [{
+        key: 'forceDownload',
+        value: function forceDownload(blob, filename) {
+            var url = (window.URL || window.webkitURL).createObjectURL(blob);
+            var link = window.document.createElement('a');
+            link.href = url;
+            link.download = filename || 'output.wav';
+            var click = document.createEvent("Event");
+            click.initEvent("click", true, true);
+            link.dispatchEvent(click);
+        }
+    }]);
+
+    return Recorder;
+})();
+
+exports.default = Recorder;
+
+},{"inline-worker":3}],3:[function(require,module,exports){
+"use strict";
+
+module.exports = require("./inline-worker");
+},{"./inline-worker":4}],4:[function(require,module,exports){
+(function (global){
+"use strict";
+
+var _createClass = (function () { function defineProperties(target, props) { for (var key in props) { var prop = props[key]; prop.configurable = true; if (prop.value) prop.writable = true; } Object.defineProperties(target, props); } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; })();
+
+var _classCallCheck = function (instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } };
+
+var WORKER_ENABLED = !!(global === global.window && global.URL && global.Blob && global.Worker);
+
+var InlineWorker = (function () {
+  function InlineWorker(func, self) {
+    var _this = this;
+
+    _classCallCheck(this, InlineWorker);
+
+    if (WORKER_ENABLED) {
+      var functionBody = func.toString().trim().match(/^function\s*\w*\s*\([\w\s,]*\)\s*{([\w\W]*?)}$/)[1];
+      var url = global.URL.createObjectURL(new global.Blob([functionBody], { type: "text/javascript" }));
+
+      return new global.Worker(url);
+    }
+
+    this.self = self;
+    this.self.postMessage = function (data) {
+      setTimeout(function () {
+        _this.onmessage({ data: data });
+      }, 0);
+    };
+
+    setTimeout(function () {
+      func.call(self);
+    }, 0);
+  }
+
+  _createClass(InlineWorker, {
+    postMessage: {
+      value: function postMessage(data) {
+        var _this = this;
+
+        setTimeout(function () {
+          _this.self.onmessage({ data: data });
+        }, 0);
+      }
+    }
+  });
+
+  return InlineWorker;
+})();
+
+module.exports = InlineWorker;
+}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {})
+},{}]},{},[1])(1)
+});