diff --git a/README.md b/README.md index bae255a660..cb47a8edd4 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,15 @@ Codegen for Rust is built in. Codegen for Python uses, protoc --python_out=cycle_gan protos/audio.proto ``` +Results +------- +Currently there is 4.39 seconds of delay between speaking and generated output +with the sidecar setup on my desktop computer. This is really great and seems +promising. + +This gets up to 6.0 seconds later. Drift continues to accrue, but it's a slow +build. + Resources --------- diff --git a/cycle_gan/protos/audio_pb2.py b/cycle_gan/protos/audio_pb2.py index 670b969a81..f77b585fe4 100644 --- a/cycle_gan/protos/audio_pb2.py +++ b/cycle_gan/protos/audio_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='protos/audio.proto', package='voder_audio', syntax='proto3', - serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xcf\x01\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x13\n\x0bsample_rate\x18\x02 \x01(\x05\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x13\n\x0boutput_rate\x18\x07 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3') + serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xec\x01\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x13\n\x0bsample_rate\x18\x02 \x01(\x05\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x13\n\x0boutput_rate\x18\x07 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x12\x1b\n\x13model_sampling_rate\x18\t \x01(\x05\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -89,6 +89,13 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + _descriptor.FieldDescriptor( + name='model_sampling_rate', full_name='voder_audio.VocodeAudioRequest.model_sampling_rate', index=8, + number=9, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -102,7 +109,7 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor( oneofs=[ ], serialized_start=36, - serialized_end=243, + serialized_end=272, ) @@ -132,8 +139,8 @@ _VOCODEAUDIORESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=245, - serialized_end=287, + serialized_start=274, + serialized_end=316, ) DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST diff --git a/cycle_gan/sidecar.py b/cycle_gan/sidecar.py index 5d5fc4d03a..170f2dba73 100755 --- a/cycle_gan/sidecar.py +++ b/cycle_gan/sidecar.py @@ -53,11 +53,17 @@ class Converter(): self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] self.logf0s_std_B = self.logf0s_normalization_params['std_B'] - def convert_partial(self, wav, conversion_direction='A2B'): + def convert_partial(self, wav, conversion_direction='A2B', model_sampling_rate=16000): + # TODO FIXME + self.sampling_rate = model_sampling_rate + wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4) + + librosa.output.write_wav('wav_padding.wav', wav, self.sampling_rate) + f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period) @@ -88,15 +94,14 @@ class Converter(): frame_period = self.frame_period) # For debugging model output, uncomment the following line: - # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) + #librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) - """ # TODO: Perhaps ditch this. It's probably unnecessary work. upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) pcm_data = upsampled.astype(np.float64) - stereo_pcm_data = np.tile(pcm_data, (2,1)).T - return stereo_pcm_data.astype(np.float32) - """ + #stereo_pcm_data = np.tile(pcm_data, (2,1)).T + #return stereo_pcm_data.astype(np.float32) + librosa.output.write_wav('model_output.wav', pcm_data, 48000) #return wav return wav_transformed @@ -126,7 +131,7 @@ def temp_file_name(suffix='.wav'): return os.path.join(TEMP_DIR.name, name) def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, discard_vocoded_audio=False, - source_rate=44100, output_rate=44100): + source_rate=44100, output_rate=44100, model_sampling_rate=16000): #audio = np.array(audio, dtype=np.int16) #data, samplerate = soundfile.read(audio) #print('samplerate', samplerate) @@ -175,7 +180,7 @@ def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, dis if skip_vocode: return audio - results = converter.convert_partial(audio, conversion_direction = 'A2B') + results = converter.convert_partial(audio, conversion_direction='A2B', model_sampling_rate=model_sampling_rate) print('results.type', type(results)) print('results.len', len(results)) @@ -218,6 +223,7 @@ def main(): results = convert(queue, source_rate=vocode_request.sample_rate, output_rate=vocode_request.output_rate, + model_sampling_rate=vocode_request.model_sampling_rate, skip_vocode=vocode_request.skip_vocode, skip_resample=vocode_request.skip_resample, save_files=vocode_request.save_files, diff --git a/protos/audio.proto b/protos/audio.proto index e9b41f4588..334ed67380 100644 --- a/protos/audio.proto +++ b/protos/audio.proto @@ -22,6 +22,9 @@ message VocodeAudioRequest { // Discard the vocoded audio and return the original bool discard_vocoded_audio = 8; + + // Rate param for the model + int32 model_sampling_rate = 9; } message VocodeAudioResponse { diff --git a/src/main.rs b/src/main.rs index e2f2756ec7..c158e095dc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -103,6 +103,7 @@ fn run_cpal_audio() -> Result<(), failure::Error> { let mut vocode_request = VocodeAudioRequest::default(); vocode_request.sample_rate = 16000; vocode_request.output_rate = 16000; + vocode_request.model_sampling_rate = 88000; vocode_request.skip_resample = false; vocode_request.skip_vocode = false; vocode_request.discard_vocoded_audio = false; @@ -199,14 +200,7 @@ fn run_cpal_audio() -> Result<(), failure::Error> { Some(mut drained) => { for (i, sample) in buffer.iter_mut().enumerate() { *sample = drained.get(i).copied().unwrap(); - /*let val = drained.pop(); - *sample = match val { - None => { - println!("Couldn't drain at index: {}", i); - 0.0 - }, - Some(d) => d * 0.3, - };*/ + //*sample = 0.0; } }, } diff --git a/src/protos/voder_audio.rs b/src/protos/voder_audio.rs index 63aabe869c..ba993a3a16 100644 --- a/src/protos/voder_audio.rs +++ b/src/protos/voder_audio.rs @@ -22,6 +22,9 @@ pub struct VocodeAudioRequest { /// Discard the vocoded audio and return the original #[prost(bool, tag="8")] pub discard_vocoded_audio: bool, + /// Rate param for the model + #[prost(int32, tag="9")] + pub model_sampling_rate: i32, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct VocodeAudioResponse {