THIS EVALUATES THE MODEL IN REALTIME +3-6sec delay

Still doesn't sound right. Probably a sample rate issue.

README.md

@@ -78,6 +78,15 @@ Codegen for Rust is built in. Codegen for Python uses,
 protoc --python_out=cycle_gan protos/audio.proto
 ```
 
+Results
+-------
+Currently there is 4.39 seconds of delay between speaking and generated output
+with the sidecar setup on my desktop computer. This is really great and seems
+promising.
+
+This gets up to 6.0 seconds later. Drift continues to accrue, but it's a slow 
+build.
+
 Resources
 ---------
 

cycle_gan/protos/audio_pb2.py

@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   name='protos/audio.proto',
   package='voder_audio',
   syntax='proto3',
-  serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xcf\x01\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x13\n\x0bsample_rate\x18\x02 \x01(\x05\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x13\n\x0boutput_rate\x18\x07 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3')
+  serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xec\x01\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x13\n\x0bsample_rate\x18\x02 \x01(\x05\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x13\n\x0boutput_rate\x18\x07 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x12\x1b\n\x13model_sampling_rate\x18\t \x01(\x05\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3')
 )
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
@@ -89,6 +89,13 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
+    _descriptor.FieldDescriptor(
+      name='model_sampling_rate', full_name='voder_audio.VocodeAudioRequest.model_sampling_rate', index=8,
+      number=9, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
   ],
   extensions=[
   ],
@@ -102,7 +109,7 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor(
   oneofs=[
   ],
   serialized_start=36,
-  serialized_end=243,
+  serialized_end=272,
 )
 
 
@@ -132,8 +139,8 @@ _VOCODEAUDIORESPONSE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=245,
-  serialized_end=287,
+  serialized_start=274,
+  serialized_end=316,
 )
 
 DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST

cycle_gan/sidecar.py

@@ -53,11 +53,17 @@ class Converter():
         self.logf0s_mean_B = self.logf0s_normalization_params['mean_B']
         self.logf0s_std_B = self.logf0s_normalization_params['std_B']
 
-    def convert_partial(self, wav, conversion_direction='A2B'):
+    def convert_partial(self, wav, conversion_direction='A2B', model_sampling_rate=16000):
+        # TODO FIXME
+        self.sampling_rate = model_sampling_rate
+
         wav = wav_padding(wav = wav,
             sr = self.sampling_rate,
             frame_period = self.frame_period,
             multiple = 4)
+
+        librosa.output.write_wav('wav_padding.wav', wav, self.sampling_rate)
+
         f0, timeaxis, sp, ap = world_decompose(wav = wav,
             fs = self.sampling_rate,
             frame_period = self.frame_period)
@@ -88,15 +94,14 @@ class Converter():
             frame_period = self.frame_period)
 
         # For debugging model output, uncomment the following line:
-        # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate)
+        #librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate)
 
-        """
         # TODO: Perhaps ditch this. It's probably unnecessary work.
         upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000)
         pcm_data = upsampled.astype(np.float64)
-        stereo_pcm_data = np.tile(pcm_data, (2,1)).T
-        return stereo_pcm_data.astype(np.float32)
-        """
+        #stereo_pcm_data = np.tile(pcm_data, (2,1)).T
+        #return stereo_pcm_data.astype(np.float32)
+        librosa.output.write_wav('model_output.wav', pcm_data, 48000)
 
         #return wav
         return wav_transformed
@@ -126,7 +131,7 @@ def temp_file_name(suffix='.wav'):
     return os.path.join(TEMP_DIR.name, name)
 
 def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, discard_vocoded_audio=False,
-            source_rate=44100, output_rate=44100):
+            source_rate=44100, output_rate=44100, model_sampling_rate=16000):
     #audio = np.array(audio, dtype=np.int16)
     #data, samplerate = soundfile.read(audio)
     #print('samplerate', samplerate)
@@ -175,7 +180,7 @@ def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, dis
     if skip_vocode:
         return audio
 
-    results = converter.convert_partial(audio, conversion_direction = 'A2B')
+    results = converter.convert_partial(audio, conversion_direction='A2B', model_sampling_rate=model_sampling_rate)
 
     print('results.type', type(results))
     print('results.len', len(results))
@@ -218,6 +223,7 @@ def main():
             results = convert(queue,
                               source_rate=vocode_request.sample_rate,
                               output_rate=vocode_request.output_rate,
+                              model_sampling_rate=vocode_request.model_sampling_rate,
                               skip_vocode=vocode_request.skip_vocode,
                               skip_resample=vocode_request.skip_resample,
                               save_files=vocode_request.save_files,

protos/audio.proto

@@ -22,6 +22,9 @@ message VocodeAudioRequest {
 
   // Discard the vocoded audio and return the original
   bool discard_vocoded_audio = 8;
+
+  // Rate param for the model
+  int32 model_sampling_rate = 9;
 }
 
 message VocodeAudioResponse {

src/main.rs

@@ -103,6 +103,7 @@ fn run_cpal_audio() -> Result<(), failure::Error> {
       let mut vocode_request = VocodeAudioRequest::default();
       vocode_request.sample_rate = 16000;
       vocode_request.output_rate = 16000;
+      vocode_request.model_sampling_rate = 88000;
       vocode_request.skip_resample = false;
       vocode_request.skip_vocode = false;
       vocode_request.discard_vocoded_audio = false;
@@ -199,14 +200,7 @@ fn run_cpal_audio() -> Result<(), failure::Error> {
           Some(mut drained) => {
             for (i, sample) in buffer.iter_mut().enumerate() {
               *sample = drained.get(i).copied().unwrap();
-              /*let val = drained.pop();
-              *sample = match val {
-                None => {
-                  println!("Couldn't drain at index: {}", i);
-                  0.0
-                },
-                Some(d) => d * 0.3,
-              };*/
+              //*sample = 0.0;
             }
           },
         }

src/protos/voder_audio.rs

@@ -22,6 +22,9 @@ pub struct VocodeAudioRequest {
     /// Discard the vocoded audio and return the original
     #[prost(bool, tag="8")]
     pub discard_vocoded_audio: bool,
+    /// Rate param for the model
+    #[prost(int32, tag="9")]
+    pub model_sampling_rate: i32,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct VocodeAudioResponse {