From 96a5ee4dabd22fe7d22810828cbb6617ab579a93 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 4 Jun 2020 00:35:57 -0400 Subject: [PATCH] This _almost_ works. I need to encode straight to i64 --- .../src/model/arpabet_glow_tts_model.rs | 236 ++++++++++++++---- tts_service/src/model/pipelines.rs | 6 +- tts_service/src/text/arpabet.rs | 103 ++++++++ 3 files changed, 293 insertions(+), 52 deletions(-) diff --git a/tts_service/src/model/arpabet_glow_tts_model.rs b/tts_service/src/model/arpabet_glow_tts_model.rs index 8d175d8398..9e97d6f1e1 100644 --- a/tts_service/src/model/arpabet_glow_tts_model.rs +++ b/tts_service/src/model/arpabet_glow_tts_model.rs @@ -4,6 +4,7 @@ use std::path::Path; use std::thread; use std::time::Duration; use tch::Tensor; +use crate::text::arpabet::text_to_arpabet_encoding_glow_tts; pub struct ArpabetGlowTtsModel { model_container: ModelContainer, @@ -17,64 +18,45 @@ impl ArpabetGlowTtsModel { }) } - pub fn encoded_arpabet_to_mel(&self, text_buffer: &Vec) -> Tensor { - let text_tensor = Tensor::of_slice(text_buffer.as_slice()); - println!("Text tensor: {:?}", text_tensor); + pub fn encoded_arpabet_to_mel(&self, arpabet_sentence: &str) -> Tensor { + println!("\narpabet sentence: {}\n", arpabet_sentence); - let text_tensor = text_tensor.unsqueeze(0); - println!("Text tensor unsq: {:?}", text_tensor); + let split = arpabet_sentence.split_whitespace(); - let lengths_tensor = Tensor::of_slice(&[text_buffer.len() as i64]); + let mut encodings = Vec::new(); // TODO EFFICIENCY + + for item in split.into_iter() { + let encoding = symbol_to_encoding(item); + if encoding > -1 { + encodings.push(encoding); + encodings.push(11); // Space + } + } + + if encodings.len() > 0 { + encodings.pop(); + } + + println!("Encodings: {:?}", encodings); /* + @DH@IH1@S @IH1@Z @DH@AH0 @S@AO1@NG @DH@AE1@T @N@EH1@V@ER0 @EH1@N@D@Z - "this is the song that never ends" - - x_tst tensor([[ 91, 109, 131, 11, 109, 146, 11, 91, 73, 11, 131, 78, 120, 11, - 91, 70, 133, 11, 119, 94, 143, 97, 11, 94, 119, 90, 146]], + x_tst tensor([[ + 91, 109, 131, + 11, 109, 146, + 11, 91, 73, + 11, 131, 78, 120, + 11, 91, 70, 133, + 11, 119, 94, 143, 97, + 11, 94, 119, 90, 146]], device='cuda:0') - - x_tst_lengths tensor([27], device='cuda:0') - */ - - /* - r: Expected tensor for argument #1 \'indices\' to have scalar type Long; - but got CPUIntType instead (while checking arguments for embedding)\n" }', - tts_service/src/model/model_container.rs:33:5 - - r: Expected tensor for argument #1 \'indices\' to have scalar type Long; - but got CPUFloatType instead (while checking arguments for embedding)\n" }', - tts_service/src/model/model_container.rs:33:5 - */ - - /* - - tst_stn 39 testing this text to speech generator - sequence 1 [[ 11 133 94 131 133 108 120 11 91 109 131 11 133 94 116 131 133 11 - 133 141 11 131 129 113 89 11 115 94 119 97 103 133 97 11]] - - x_tst 1 tensor([[ 11, 133, 94, 131, 133, 108, 120, 11, 91, 109, 131, 11, 133, 94, - 116, 131, 133, 11, 133, 141, 11, 131, 129, 113, 89, 11, 115, 94, - 119, 97, 103, 133, 97, 11]]) - - x_tst_lengths 1 tensor([34]) - - */ - // let text : Vec = vec![ 91, 109, 131, 11, 109, 146, 11, 91, 73, 11, 131, 78, 120, 11, - // 91, 70, 133, 11, 119, 94, 143, 97, 11, 94, 119, 90, 146]; - - let text : Vec = vec![11, 133, 94, 131, 133, 108, 120, 11, 91, 109, 131, - 11, 133, 94, 116, 131, 133, 11, 133, 141, 11, 131, 129, 113, 89, 11, 115, 94, - 119, 97, 103, 133, 97, 11]; + //let text : Vec = vec![118, 86, 11, 119, 102, 118, 11, 109, 146, 11, 90, 66, 119, 73, 117, 90, 11, 133, 130, 74, 118, 129, 11, 73, 119, 90, 11, 86, 11, 102, 70, 118, 11, 91, 73, 11, 129, 130, 94, 146, 108, 90, 73, 119, 133, 11, 74, 143, 11, 91, 73, 11, 145, 140, 119, 86, 133, 73, 90, 11, 56, 57, 38, 57, 42, 56, 6, 11, 145, 141, 11, 132, 137, 90, 11, 130, 108, 131, 129, 94, 116, 133, 11, 118, 113]; - //let text : Vec = text.iter().map(|n| *n as i64).collect(); - //let one_d : Vec<&[i64]> = vec![text.as_slice()]; - //let two_d : &[&[i64]] = one_d.as_slice(); - - let x_tst = Tensor::of_slice(&text.as_slice()); + let x_tst = Tensor::of_slice(&encodings.as_slice()); println!("\n\nTensor size: {:?}", x_tst.size()); @@ -83,9 +65,165 @@ impl ArpabetGlowTtsModel { println!("\n\nTensor unsqueezed size: {:?}", x_tst_2.size()); //let lengths = [27i64]; - let lengths = [34i64]; + let lengths = [encodings.len() as i64]; let x_tst_lengths = Tensor::of_slice(&lengths); self.model_container.forward2(&x_tst_2, &x_tst_lengths) } } + +// TODO: This is gross af +fn symbol_to_encoding(symbol: &str) -> i64 { + match (symbol) { + "_" => 0, + "-" => 1, + "!" => 2, + "'" => 3, + "(" => 4, + ")" => 5, + "," => 6, + "." => 7, + ":" => 8, + ";" => 9, + "?" => 10, + " " => 11, + "A" => 12, + "B" => 13, + "C" => 14, + "D" => 15, + "E" => 16, + "F" => 17, + "G" => 18, + "H" => 19, + "I" => 20, + "J" => 21, + "K" => 22, + "L" => 23, + "M" => 24, + "N" => 25, + "O" => 26, + "P" => 27, + "Q" => 28, + "R" => 29, + "S" => 30, + "T" => 31, + "U" => 32, + "V" => 33, + "W" => 34, + "X" => 35, + "Y" => 36, + "Z" => 37, + "a" => 38, + "b" => 39, + "c" => 40, + "d" => 41, + "e" => 42, + "f" => 43, + "g" => 44, + "h" => 45, + "i" => 46, + "j" => 47, + "k" => 48, + "l" => 49, + "m" => 50, + "n" => 51, + "o" => 52, + "p" => 53, + "q" => 54, + "r" => 55, + "s" => 56, + "t" => 57, + "u" => 58, + "v" => 59, + "w" => 60, + "x" => 61, + "y" => 62, + "z" => 63, + "@AA" => 64, + "@AA0" => 65, + "@AA1" => 66, + "@AA2" => 67, + "@AE" => 68, + "@AE0" => 69, + "@AE1" => 70, + "@AE2" => 71, + "@AH" => 72, + "@AH0" => 73, + "@AH1" => 74, + "@AH2" => 75, + "@AO" => 76, + "@AO0" => 77, + "@AO1" => 78, + "@AO2" => 79, + "@AW" => 80, + "@AW0" => 81, + "@AW1" => 82, + "@AW2" => 83, + "@AY" => 84, + "@AY0" => 85, + "@AY1" => 86, + "@AY2" => 87, + "@B" => 88, + "@CH" => 89, + "@D" => 90, + "@DH" => 91, + "@EH" => 92, + "@EH0" => 93, + "@EH1" => 94, + "@EH2" => 95, + "@ER" => 96, + "@ER0" => 97, + "@ER1" => 98, + "@ER2" => 99, + "@EY" => 100, + "@EY0" => 101, + "@EY1" => 102, + "@EY2" => 103, + "@F" => 104, + "@G" => 105, + "@HH" => 106, + "@IH" => 107, + "@IH0" => 108, + "@IH1" => 109, + "@IH2" => 110, + "@IY" => 111, + "@IY0" => 112, + "@IY1" => 113, + "@IY2" => 114, + "@JH" => 115, + "@K" => 116, + "@L" => 117, + "@M" => 118, + "@N" => 119, + "@NG" => 120, + "@OW" => 121, + "@OW0" => 122, + "@OW1" => 123, + "@OW2" => 124, + "@OY" => 125, + "@OY0" => 126, + "@OY1" => 127, + "@OY2" => 128, + "@P" => 129, + "@R" => 130, + "@S" => 131, + "@SH" => 132, + "@T" => 133, + "@TH" => 134, + "@UH" => 135, + "@UH0" => 136, + "@UH1" => 137, + "@UH2" => 138, + "@UW" => 139, + "@UW0" => 140, + "@UW1" => 141, + "@UW2" => 142, + "@V" => 143, + "@W" => 144, + "@Y" => 145, + "@Z" => 146, + "@ZH" => 147, + _ => -1, + } +} + diff --git a/tts_service/src/model/pipelines.rs b/tts_service/src/model/pipelines.rs index ba9b472265..11966f4125 100644 --- a/tts_service/src/model/pipelines.rs +++ b/tts_service/src/model/pipelines.rs @@ -2,7 +2,7 @@ use arpabet::Arpabet; use crate::model::arpabet_tacotron_model::ArpabetTacotronModel; use crate::model::melgan_model::MelganModel; use crate::model::old_model::TacoMelModel; -use crate::text::arpabet::text_to_arpabet_encoding; +use crate::text::arpabet::{text_to_arpabet_encoding, text_to_arpabet_encoding_glow_tts}; use hound::SampleFormat; use hound::WavSpec; use hound::WavWriter; @@ -20,9 +20,9 @@ pub fn arpabet_glow_tts_melgan_pipeline( melgan: &MelganModel) -> Option> { let arpabet = Arpabet::load_cmudict(); // TODO: Inefficient. - let encoded = text_to_arpabet_encoding(arpabet, &cleaned_text); + let arpabet_sentence = text_to_arpabet_encoding_glow_tts(arpabet, &cleaned_text); - let mel_tensor = arpabet_glow_tts.encoded_arpabet_to_mel(&encoded); + let mel_tensor = arpabet_glow_tts.encoded_arpabet_to_mel(&arpabet_sentence); let audio_tensor = melgan.tacotron_mel_to_audio(&mel_tensor); let audio_signal = mel_audio_tensor_to_audio_signal(audio_tensor); diff --git a/tts_service/src/text/arpabet.rs b/tts_service/src/text/arpabet.rs index 9ca6a97a1f..6e2840fde1 100644 --- a/tts_service/src/text/arpabet.rs +++ b/tts_service/src/text/arpabet.rs @@ -22,6 +22,109 @@ pub fn text_to_standard_encoding(text: &str) -> Vec { text_buffer } +/// This is the encoding that Glow TTS uses. +pub fn text_to_arpabet_encoding_glow_tts(arpabet: &Arpabet, text: &str) -> String { + let sentence_tokenizer = SentenceTokenizer::new(); + let numerics = Numerics::default(); + + let text = text.to_ascii_lowercase(); + + let tokens = sentence_tokenizer.tokenize(&text); + info!("Sentence Tokens: {:?}", tokens); + + let mut needs_space = false; + + let mut transformed_sentence = String::new(); + + for token in tokens { + match token { + Token::ApostrophenatedWord(word) + | Token::HyphenatedWord(word) + | Token::Word(word) => { + if needs_space { + transformed_sentence.push_str(" "); + needs_space = false; + } + + // Option + //pub type Polyphone = Vec; + // + match arpabet.get_polyphone(&word) { + None => { + transformed_sentence.push_str(&word); + needs_space = true; + }, + Some(arpas) => { + let collected : Vec= arpas.iter() + .map(|a| ["@", a.to_str()].join("")) // TODO: Add to arpabet library + .collect(); + + let transformed = collected.join(""); + + transformed_sentence.push_str(&transformed); + needs_space = true; + }, + } + }, + Token::Integer(integer_string) => { + if needs_space { + transformed_sentence.push_str(" "); + needs_space = false; + } + // TODO + /*let number_words = integer_string.parse::() + .ok() + .and_then(|number| numerics.convert_number(number).ok()) + .unwrap_or(Vec::new());*/ + + /*for word in number_words { + if needs_space { + encoded_buffer.push(space); + needs_space = false; + } + if arpabet_lookup_and_encode_word(&word, &arpabet, &mut encoded_buffer) { + needs_space = true; + } + }*/ + }, + Token::Punctuation(punctuation) => { + match punctuation { + Punctuation::Period => { + transformed_sentence.push_str("."); + needs_space = true; + }, + Punctuation::Comma | Punctuation::Semicolon => { + transformed_sentence.push_str(","); + needs_space = true; + }, + Punctuation::Question => { + transformed_sentence.push_str("?"); + needs_space = true; + }, + Punctuation::Exclamation => { + transformed_sentence.push_str("!"); + needs_space = true; + }, + Punctuation::Colon | Punctuation::Dash => { + transformed_sentence.push_str(" - "); + needs_space = false; + }, + } + }, + // TODO: + Token::RealNumber(_) => continue, + Token::CommaFormattedInteger(_) => continue, + Token::CommaFormattedRealNumber(_) => continue, + Token::Hashtag(_) => continue, + Token::UsernameMention(_) => continue, + Token::Url(_) => continue, + Token::Unknown(_) => continue, // skip, we don't know what this is yet. + } + } + + transformed_sentence +} + /// Convert text to the encoding used in my Arpabet extension pub fn text_to_arpabet_encoding(arpabet: &Arpabet, text: &str) -> Vec { let sentence_tokenizer = SentenceTokenizer::new();