This _almost_ works. I need to encode straight to i64

This commit is contained in:
Brandon Thomas committed 2020-06-04 00:35:57 -04:00
1 parent a6af26b4df
commit 96a5ee4dab
3 files changed
+293 -52

No files matched your search

+187 -49
View File
@@ -4,6 +4,7 @@ use std::path::Path;
use std::thread;
use std::time::Duration;
use tch::Tensor;
use crate::text::arpabet::text_to_arpabet_encoding_glow_tts;
pub struct ArpabetGlowTtsModel {
model_container: ModelContainer,
@@ -17,64 +18,45 @@ impl ArpabetGlowTtsModel {
})
}
pub fn encoded_arpabet_to_mel(&self, text_buffer: &Vec<i64>) -> Tensor {
let text_tensor = Tensor::of_slice(text_buffer.as_slice());
println!("Text tensor: {:?}", text_tensor);
pub fn encoded_arpabet_to_mel(&self, arpabet_sentence: &str) -> Tensor {
println!("\narpabet sentence: {}\n", arpabet_sentence);
let text_tensor = text_tensor.unsqueeze(0);
println!("Text tensor unsq: {:?}", text_tensor);
let split = arpabet_sentence.split_whitespace();
let lengths_tensor = Tensor::of_slice(&[text_buffer.len() as i64]);
let mut encodings = Vec::new(); // TODO EFFICIENCY
for item in split.into_iter() {
let encoding = symbol_to_encoding(item);
if encoding > -1 {
encodings.push(encoding);
encodings.push(11); // Space
}
}
if encodings.len() > 0 {
encodings.pop();
}
println!("Encodings: {:?}", encodings);
/*
@DH@IH1@S @IH1@Z @DH@AH0 @S@AO1@NG @DH@AE1@T @N@EH1@V@ER0 @EH1@N@D@Z
"this is the song that never ends"
x_tst tensor([[ 91, 109, 131, 11, 109, 146, 11, 91, 73, 11, 131, 78, 120, 11,
91, 70, 133, 11, 119, 94, 143, 97, 11, 94, 119, 90, 146]],
x_tst tensor([[
91, 109, 131,
11, 109, 146,
11, 91, 73,
11, 131, 78, 120,
11, 91, 70, 133,
11, 119, 94, 143, 97,
11, 94, 119, 90, 146]],
device='cuda:0')
x_tst_lengths tensor([27], device='cuda:0')
*/
/*
r: Expected tensor for argument #1 \'indices\' to have scalar type Long;
but got CPUIntType instead (while checking arguments for embedding)\n" }',
tts_service/src/model/model_container.rs:33:5
r: Expected tensor for argument #1 \'indices\' to have scalar type Long;
but got CPUFloatType instead (while checking arguments for embedding)\n" }',
tts_service/src/model/model_container.rs:33:5
*/
/*
tst_stn 39 testing this text to speech generator
sequence 1 [[ 11 133 94 131 133 108 120 11 91 109 131 11 133 94 116 131 133 11
133 141 11 131 129 113 89 11 115 94 119 97 103 133 97 11]]
x_tst 1 tensor([[ 11, 133, 94, 131, 133, 108, 120, 11, 91, 109, 131, 11, 133, 94,
116, 131, 133, 11, 133, 141, 11, 131, 129, 113, 89, 11, 115, 94,
119, 97, 103, 133, 97, 11]])
x_tst_lengths 1 tensor([34])
*/
// let text : Vec<i64> = vec![ 91, 109, 131, 11, 109, 146, 11, 91, 73, 11, 131, 78, 120, 11,
// 91, 70, 133, 11, 119, 94, 143, 97, 11, 94, 119, 90, 146];
let text : Vec<i64> = vec![11, 133, 94, 131, 133, 108, 120, 11, 91, 109, 131,
11, 133, 94, 116, 131, 133, 11, 133, 141, 11, 131, 129, 113, 89, 11, 115, 94,
119, 97, 103, 133, 97, 11];
//let text : Vec<i64> = vec![118, 86, 11, 119, 102, 118, 11, 109, 146, 11, 90, 66, 119, 73, 117, 90, 11, 133, 130, 74, 118, 129, 11, 73, 119, 90, 11, 86, 11, 102, 70, 118, 11, 91, 73, 11, 129, 130, 94, 146, 108, 90, 73, 119, 133, 11, 74, 143, 11, 91, 73, 11, 145, 140, 119, 86, 133, 73, 90, 11, 56, 57, 38, 57, 42, 56, 6, 11, 145, 141, 11, 132, 137, 90, 11, 130, 108, 131, 129, 94, 116, 133, 11, 118, 113];
//let text : Vec<f32> = text.iter().map(|n| *n as i64).collect();
//let one_d : Vec<&[i64]> = vec![text.as_slice()];
//let two_d : &[&[i64]] = one_d.as_slice();
let x_tst = Tensor::of_slice(&text.as_slice());
let x_tst = Tensor::of_slice(&encodings.as_slice());
println!("\n\nTensor size: {:?}", x_tst.size());
@@ -83,9 +65,165 @@ impl ArpabetGlowTtsModel {
println!("\n\nTensor unsqueezed size: {:?}", x_tst_2.size());
//let lengths = [27i64];
let lengths = [34i64];
let lengths = [encodings.len() as i64];
let x_tst_lengths = Tensor::of_slice(&lengths);
self.model_container.forward2(&x_tst_2, &x_tst_lengths)
}
}
// TODO: This is gross af
fn symbol_to_encoding(symbol: &str) -> i64 {
match (symbol) {
"_" => 0,
"-" => 1,
"!" => 2,
"'" => 3,
"(" => 4,
")" => 5,
"," => 6,
"." => 7,
":" => 8,
";" => 9,
"?" => 10,
" " => 11,
"A" => 12,
"B" => 13,
"C" => 14,
"D" => 15,
"E" => 16,
"F" => 17,
"G" => 18,
"H" => 19,
"I" => 20,
"J" => 21,
"K" => 22,
"L" => 23,
"M" => 24,
"N" => 25,
"O" => 26,
"P" => 27,
"Q" => 28,
"R" => 29,
"S" => 30,
"T" => 31,
"U" => 32,
"V" => 33,
"W" => 34,
"X" => 35,
"Y" => 36,
"Z" => 37,
"a" => 38,
"b" => 39,
"c" => 40,
"d" => 41,
"e" => 42,
"f" => 43,
"g" => 44,
"h" => 45,
"i" => 46,
"j" => 47,
"k" => 48,
"l" => 49,
"m" => 50,
"n" => 51,
"o" => 52,
"p" => 53,
"q" => 54,
"r" => 55,
"s" => 56,
"t" => 57,
"u" => 58,
"v" => 59,
"w" => 60,
"x" => 61,
"y" => 62,
"z" => 63,
"@AA" => 64,
"@AA0" => 65,
"@AA1" => 66,
"@AA2" => 67,
"@AE" => 68,
"@AE0" => 69,
"@AE1" => 70,
"@AE2" => 71,
"@AH" => 72,
"@AH0" => 73,
"@AH1" => 74,
"@AH2" => 75,
"@AO" => 76,
"@AO0" => 77,
"@AO1" => 78,
"@AO2" => 79,
"@AW" => 80,
"@AW0" => 81,
"@AW1" => 82,
"@AW2" => 83,
"@AY" => 84,
"@AY0" => 85,
"@AY1" => 86,
"@AY2" => 87,
"@B" => 88,
"@CH" => 89,
"@D" => 90,
"@DH" => 91,
"@EH" => 92,
"@EH0" => 93,
"@EH1" => 94,
"@EH2" => 95,
"@ER" => 96,
"@ER0" => 97,
"@ER1" => 98,
"@ER2" => 99,
"@EY" => 100,
"@EY0" => 101,
"@EY1" => 102,
"@EY2" => 103,
"@F" => 104,
"@G" => 105,
"@HH" => 106,
"@IH" => 107,
"@IH0" => 108,
"@IH1" => 109,
"@IH2" => 110,
"@IY" => 111,
"@IY0" => 112,
"@IY1" => 113,
"@IY2" => 114,
"@JH" => 115,
"@K" => 116,
"@L" => 117,
"@M" => 118,
"@N" => 119,
"@NG" => 120,
"@OW" => 121,
"@OW0" => 122,
"@OW1" => 123,
"@OW2" => 124,
"@OY" => 125,
"@OY0" => 126,
"@OY1" => 127,
"@OY2" => 128,
"@P" => 129,
"@R" => 130,
"@S" => 131,
"@SH" => 132,
"@T" => 133,
"@TH" => 134,
"@UH" => 135,
"@UH0" => 136,
"@UH1" => 137,
"@UH2" => 138,
"@UW" => 139,
"@UW0" => 140,
"@UW1" => 141,
"@UW2" => 142,
"@V" => 143,
"@W" => 144,
"@Y" => 145,
"@Z" => 146,
"@ZH" => 147,
_ => -1,
}
}
+3 -3
View File
@@ -2,7 +2,7 @@ use arpabet::Arpabet;
use crate::model::arpabet_tacotron_model::ArpabetTacotronModel;
use crate::model::melgan_model::MelganModel;
use crate::model::old_model::TacoMelModel;
use crate::text::arpabet::text_to_arpabet_encoding;
use crate::text::arpabet::{text_to_arpabet_encoding, text_to_arpabet_encoding_glow_tts};
use hound::SampleFormat;
use hound::WavSpec;
use hound::WavWriter;
@@ -20,9 +20,9 @@ pub fn arpabet_glow_tts_melgan_pipeline(
melgan: &MelganModel) -> Option<Vec<u8>> {
let arpabet = Arpabet::load_cmudict(); // TODO: Inefficient.
let encoded = text_to_arpabet_encoding(arpabet, &cleaned_text);
let arpabet_sentence = text_to_arpabet_encoding_glow_tts(arpabet, &cleaned_text);
let mel_tensor = arpabet_glow_tts.encoded_arpabet_to_mel(&encoded);
let mel_tensor = arpabet_glow_tts.encoded_arpabet_to_mel(&arpabet_sentence);
let audio_tensor = melgan.tacotron_mel_to_audio(&mel_tensor);
let audio_signal = mel_audio_tensor_to_audio_signal(audio_tensor);
+103
View File
@@ -22,6 +22,109 @@ pub fn text_to_standard_encoding(text: &str) -> Vec<i64> {
text_buffer
}
/// This is the encoding that Glow TTS uses.
pub fn text_to_arpabet_encoding_glow_tts(arpabet: &Arpabet, text: &str) -> String {
let sentence_tokenizer = SentenceTokenizer::new();
let numerics = Numerics::default();
let text = text.to_ascii_lowercase();
let tokens = sentence_tokenizer.tokenize(&text);
info!("Sentence Tokens: {:?}", tokens);
let mut needs_space = false;
let mut transformed_sentence = String::new();
for token in tokens {
match token {
Token::ApostrophenatedWord(word)
| Token::HyphenatedWord(word)
| Token::Word(word) => {
if needs_space {
transformed_sentence.push_str(" ");
needs_space = false;
}
// Option<Polyphone>
//pub type Polyphone = Vec<Phoneme>;
//
match arpabet.get_polyphone(&word) {
None => {
transformed_sentence.push_str(&word);
needs_space = true;
},
Some(arpas) => {
let collected : Vec<String>= arpas.iter()
.map(|a| ["@", a.to_str()].join("")) // TODO: Add to arpabet library
.collect();
let transformed = collected.join("");
transformed_sentence.push_str(&transformed);
needs_space = true;
},
}
},
Token::Integer(integer_string) => {
if needs_space {
transformed_sentence.push_str(" ");
needs_space = false;
}
// TODO
/*let number_words = integer_string.parse::<i64>()
.ok()
.and_then(|number| numerics.convert_number(number).ok())
.unwrap_or(Vec::new());*/
/*for word in number_words {
if needs_space {
encoded_buffer.push(space);
needs_space = false;
}
if arpabet_lookup_and_encode_word(&word, &arpabet, &mut encoded_buffer) {
needs_space = true;
}
}*/
},
Token::Punctuation(punctuation) => {
match punctuation {
Punctuation::Period => {
transformed_sentence.push_str(".");
needs_space = true;
},
Punctuation::Comma | Punctuation::Semicolon => {
transformed_sentence.push_str(",");
needs_space = true;
},
Punctuation::Question => {
transformed_sentence.push_str("?");
needs_space = true;
},
Punctuation::Exclamation => {
transformed_sentence.push_str("!");
needs_space = true;
},
Punctuation::Colon | Punctuation::Dash => {
transformed_sentence.push_str(" - ");
needs_space = false;
},
}
},
// TODO:
Token::RealNumber(_) => continue,
Token::CommaFormattedInteger(_) => continue,
Token::CommaFormattedRealNumber(_) => continue,
Token::Hashtag(_) => continue,
Token::UsernameMention(_) => continue,
Token::Url(_) => continue,
Token::Unknown(_) => continue, // skip, we don't know what this is yet.
}
}
transformed_sentence
}
/// Convert text to the encoding used in my Arpabet extension
pub fn text_to_arpabet_encoding(arpabet: &Arpabet, text: &str) -> Vec<i64> {
let sentence_tokenizer = SentenceTokenizer::new();