mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-20 16:45:10 +00:00
This _almost_ works. I need to encode straight to i64
This commit is contained in:
1 parent
a6af26b4df
commit
96a5ee4dab
3 files changed
+293
-52
No files matched your search
@@ -4,6 +4,7 @@ use std::path::Path;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use tch::Tensor;
|
||||
use crate::text::arpabet::text_to_arpabet_encoding_glow_tts;
|
||||
|
||||
pub struct ArpabetGlowTtsModel {
|
||||
model_container: ModelContainer,
|
||||
@@ -17,64 +18,45 @@ impl ArpabetGlowTtsModel {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn encoded_arpabet_to_mel(&self, text_buffer: &Vec<i64>) -> Tensor {
|
||||
let text_tensor = Tensor::of_slice(text_buffer.as_slice());
|
||||
println!("Text tensor: {:?}", text_tensor);
|
||||
pub fn encoded_arpabet_to_mel(&self, arpabet_sentence: &str) -> Tensor {
|
||||
println!("\narpabet sentence: {}\n", arpabet_sentence);
|
||||
|
||||
let text_tensor = text_tensor.unsqueeze(0);
|
||||
println!("Text tensor unsq: {:?}", text_tensor);
|
||||
let split = arpabet_sentence.split_whitespace();
|
||||
|
||||
let lengths_tensor = Tensor::of_slice(&[text_buffer.len() as i64]);
|
||||
let mut encodings = Vec::new(); // TODO EFFICIENCY
|
||||
|
||||
for item in split.into_iter() {
|
||||
let encoding = symbol_to_encoding(item);
|
||||
if encoding > -1 {
|
||||
encodings.push(encoding);
|
||||
encodings.push(11); // Space
|
||||
}
|
||||
}
|
||||
|
||||
if encodings.len() > 0 {
|
||||
encodings.pop();
|
||||
}
|
||||
|
||||
println!("Encodings: {:?}", encodings);
|
||||
|
||||
/*
|
||||
@DH@IH1@S @IH1@Z @DH@AH0 @S@AO1@NG @DH@AE1@T @N@EH1@V@ER0 @EH1@N@D@Z
|
||||
|
||||
"this is the song that never ends"
|
||||
|
||||
x_tst tensor([[ 91, 109, 131, 11, 109, 146, 11, 91, 73, 11, 131, 78, 120, 11,
|
||||
91, 70, 133, 11, 119, 94, 143, 97, 11, 94, 119, 90, 146]],
|
||||
x_tst tensor([[
|
||||
91, 109, 131,
|
||||
11, 109, 146,
|
||||
11, 91, 73,
|
||||
11, 131, 78, 120,
|
||||
11, 91, 70, 133,
|
||||
11, 119, 94, 143, 97,
|
||||
11, 94, 119, 90, 146]],
|
||||
device='cuda:0')
|
||||
|
||||
x_tst_lengths tensor([27], device='cuda:0')
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
r: Expected tensor for argument #1 \'indices\' to have scalar type Long;
|
||||
but got CPUIntType instead (while checking arguments for embedding)\n" }',
|
||||
tts_service/src/model/model_container.rs:33:5
|
||||
|
||||
r: Expected tensor for argument #1 \'indices\' to have scalar type Long;
|
||||
but got CPUFloatType instead (while checking arguments for embedding)\n" }',
|
||||
tts_service/src/model/model_container.rs:33:5
|
||||
*/
|
||||
|
||||
/*
|
||||
|
||||
tst_stn 39 testing this text to speech generator
|
||||
sequence 1 [[ 11 133 94 131 133 108 120 11 91 109 131 11 133 94 116 131 133 11
|
||||
133 141 11 131 129 113 89 11 115 94 119 97 103 133 97 11]]
|
||||
|
||||
x_tst 1 tensor([[ 11, 133, 94, 131, 133, 108, 120, 11, 91, 109, 131, 11, 133, 94,
|
||||
116, 131, 133, 11, 133, 141, 11, 131, 129, 113, 89, 11, 115, 94,
|
||||
119, 97, 103, 133, 97, 11]])
|
||||
|
||||
x_tst_lengths 1 tensor([34])
|
||||
|
||||
*/
|
||||
// let text : Vec<i64> = vec![ 91, 109, 131, 11, 109, 146, 11, 91, 73, 11, 131, 78, 120, 11,
|
||||
// 91, 70, 133, 11, 119, 94, 143, 97, 11, 94, 119, 90, 146];
|
||||
|
||||
let text : Vec<i64> = vec![11, 133, 94, 131, 133, 108, 120, 11, 91, 109, 131,
|
||||
11, 133, 94, 116, 131, 133, 11, 133, 141, 11, 131, 129, 113, 89, 11, 115, 94,
|
||||
119, 97, 103, 133, 97, 11];
|
||||
//let text : Vec<i64> = vec![118, 86, 11, 119, 102, 118, 11, 109, 146, 11, 90, 66, 119, 73, 117, 90, 11, 133, 130, 74, 118, 129, 11, 73, 119, 90, 11, 86, 11, 102, 70, 118, 11, 91, 73, 11, 129, 130, 94, 146, 108, 90, 73, 119, 133, 11, 74, 143, 11, 91, 73, 11, 145, 140, 119, 86, 133, 73, 90, 11, 56, 57, 38, 57, 42, 56, 6, 11, 145, 141, 11, 132, 137, 90, 11, 130, 108, 131, 129, 94, 116, 133, 11, 118, 113];
|
||||
|
||||
|
||||
//let text : Vec<f32> = text.iter().map(|n| *n as i64).collect();
|
||||
|
||||
//let one_d : Vec<&[i64]> = vec![text.as_slice()];
|
||||
//let two_d : &[&[i64]] = one_d.as_slice();
|
||||
|
||||
let x_tst = Tensor::of_slice(&text.as_slice());
|
||||
let x_tst = Tensor::of_slice(&encodings.as_slice());
|
||||
|
||||
println!("\n\nTensor size: {:?}", x_tst.size());
|
||||
|
||||
@@ -83,9 +65,165 @@ impl ArpabetGlowTtsModel {
|
||||
println!("\n\nTensor unsqueezed size: {:?}", x_tst_2.size());
|
||||
|
||||
//let lengths = [27i64];
|
||||
let lengths = [34i64];
|
||||
let lengths = [encodings.len() as i64];
|
||||
let x_tst_lengths = Tensor::of_slice(&lengths);
|
||||
|
||||
self.model_container.forward2(&x_tst_2, &x_tst_lengths)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: This is gross af
|
||||
fn symbol_to_encoding(symbol: &str) -> i64 {
|
||||
match (symbol) {
|
||||
"_" => 0,
|
||||
"-" => 1,
|
||||
"!" => 2,
|
||||
"'" => 3,
|
||||
"(" => 4,
|
||||
")" => 5,
|
||||
"," => 6,
|
||||
"." => 7,
|
||||
":" => 8,
|
||||
";" => 9,
|
||||
"?" => 10,
|
||||
" " => 11,
|
||||
"A" => 12,
|
||||
"B" => 13,
|
||||
"C" => 14,
|
||||
"D" => 15,
|
||||
"E" => 16,
|
||||
"F" => 17,
|
||||
"G" => 18,
|
||||
"H" => 19,
|
||||
"I" => 20,
|
||||
"J" => 21,
|
||||
"K" => 22,
|
||||
"L" => 23,
|
||||
"M" => 24,
|
||||
"N" => 25,
|
||||
"O" => 26,
|
||||
"P" => 27,
|
||||
"Q" => 28,
|
||||
"R" => 29,
|
||||
"S" => 30,
|
||||
"T" => 31,
|
||||
"U" => 32,
|
||||
"V" => 33,
|
||||
"W" => 34,
|
||||
"X" => 35,
|
||||
"Y" => 36,
|
||||
"Z" => 37,
|
||||
"a" => 38,
|
||||
"b" => 39,
|
||||
"c" => 40,
|
||||
"d" => 41,
|
||||
"e" => 42,
|
||||
"f" => 43,
|
||||
"g" => 44,
|
||||
"h" => 45,
|
||||
"i" => 46,
|
||||
"j" => 47,
|
||||
"k" => 48,
|
||||
"l" => 49,
|
||||
"m" => 50,
|
||||
"n" => 51,
|
||||
"o" => 52,
|
||||
"p" => 53,
|
||||
"q" => 54,
|
||||
"r" => 55,
|
||||
"s" => 56,
|
||||
"t" => 57,
|
||||
"u" => 58,
|
||||
"v" => 59,
|
||||
"w" => 60,
|
||||
"x" => 61,
|
||||
"y" => 62,
|
||||
"z" => 63,
|
||||
"@AA" => 64,
|
||||
"@AA0" => 65,
|
||||
"@AA1" => 66,
|
||||
"@AA2" => 67,
|
||||
"@AE" => 68,
|
||||
"@AE0" => 69,
|
||||
"@AE1" => 70,
|
||||
"@AE2" => 71,
|
||||
"@AH" => 72,
|
||||
"@AH0" => 73,
|
||||
"@AH1" => 74,
|
||||
"@AH2" => 75,
|
||||
"@AO" => 76,
|
||||
"@AO0" => 77,
|
||||
"@AO1" => 78,
|
||||
"@AO2" => 79,
|
||||
"@AW" => 80,
|
||||
"@AW0" => 81,
|
||||
"@AW1" => 82,
|
||||
"@AW2" => 83,
|
||||
"@AY" => 84,
|
||||
"@AY0" => 85,
|
||||
"@AY1" => 86,
|
||||
"@AY2" => 87,
|
||||
"@B" => 88,
|
||||
"@CH" => 89,
|
||||
"@D" => 90,
|
||||
"@DH" => 91,
|
||||
"@EH" => 92,
|
||||
"@EH0" => 93,
|
||||
"@EH1" => 94,
|
||||
"@EH2" => 95,
|
||||
"@ER" => 96,
|
||||
"@ER0" => 97,
|
||||
"@ER1" => 98,
|
||||
"@ER2" => 99,
|
||||
"@EY" => 100,
|
||||
"@EY0" => 101,
|
||||
"@EY1" => 102,
|
||||
"@EY2" => 103,
|
||||
"@F" => 104,
|
||||
"@G" => 105,
|
||||
"@HH" => 106,
|
||||
"@IH" => 107,
|
||||
"@IH0" => 108,
|
||||
"@IH1" => 109,
|
||||
"@IH2" => 110,
|
||||
"@IY" => 111,
|
||||
"@IY0" => 112,
|
||||
"@IY1" => 113,
|
||||
"@IY2" => 114,
|
||||
"@JH" => 115,
|
||||
"@K" => 116,
|
||||
"@L" => 117,
|
||||
"@M" => 118,
|
||||
"@N" => 119,
|
||||
"@NG" => 120,
|
||||
"@OW" => 121,
|
||||
"@OW0" => 122,
|
||||
"@OW1" => 123,
|
||||
"@OW2" => 124,
|
||||
"@OY" => 125,
|
||||
"@OY0" => 126,
|
||||
"@OY1" => 127,
|
||||
"@OY2" => 128,
|
||||
"@P" => 129,
|
||||
"@R" => 130,
|
||||
"@S" => 131,
|
||||
"@SH" => 132,
|
||||
"@T" => 133,
|
||||
"@TH" => 134,
|
||||
"@UH" => 135,
|
||||
"@UH0" => 136,
|
||||
"@UH1" => 137,
|
||||
"@UH2" => 138,
|
||||
"@UW" => 139,
|
||||
"@UW0" => 140,
|
||||
"@UW1" => 141,
|
||||
"@UW2" => 142,
|
||||
"@V" => 143,
|
||||
"@W" => 144,
|
||||
"@Y" => 145,
|
||||
"@Z" => 146,
|
||||
"@ZH" => 147,
|
||||
_ => -1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use arpabet::Arpabet;
|
||||
use crate::model::arpabet_tacotron_model::ArpabetTacotronModel;
|
||||
use crate::model::melgan_model::MelganModel;
|
||||
use crate::model::old_model::TacoMelModel;
|
||||
use crate::text::arpabet::text_to_arpabet_encoding;
|
||||
use crate::text::arpabet::{text_to_arpabet_encoding, text_to_arpabet_encoding_glow_tts};
|
||||
use hound::SampleFormat;
|
||||
use hound::WavSpec;
|
||||
use hound::WavWriter;
|
||||
@@ -20,9 +20,9 @@ pub fn arpabet_glow_tts_melgan_pipeline(
|
||||
melgan: &MelganModel) -> Option<Vec<u8>> {
|
||||
|
||||
let arpabet = Arpabet::load_cmudict(); // TODO: Inefficient.
|
||||
let encoded = text_to_arpabet_encoding(arpabet, &cleaned_text);
|
||||
let arpabet_sentence = text_to_arpabet_encoding_glow_tts(arpabet, &cleaned_text);
|
||||
|
||||
let mel_tensor = arpabet_glow_tts.encoded_arpabet_to_mel(&encoded);
|
||||
let mel_tensor = arpabet_glow_tts.encoded_arpabet_to_mel(&arpabet_sentence);
|
||||
|
||||
let audio_tensor = melgan.tacotron_mel_to_audio(&mel_tensor);
|
||||
let audio_signal = mel_audio_tensor_to_audio_signal(audio_tensor);
|
||||
|
||||
@@ -22,6 +22,109 @@ pub fn text_to_standard_encoding(text: &str) -> Vec<i64> {
|
||||
text_buffer
|
||||
}
|
||||
|
||||
/// This is the encoding that Glow TTS uses.
|
||||
pub fn text_to_arpabet_encoding_glow_tts(arpabet: &Arpabet, text: &str) -> String {
|
||||
let sentence_tokenizer = SentenceTokenizer::new();
|
||||
let numerics = Numerics::default();
|
||||
|
||||
let text = text.to_ascii_lowercase();
|
||||
|
||||
let tokens = sentence_tokenizer.tokenize(&text);
|
||||
info!("Sentence Tokens: {:?}", tokens);
|
||||
|
||||
let mut needs_space = false;
|
||||
|
||||
let mut transformed_sentence = String::new();
|
||||
|
||||
for token in tokens {
|
||||
match token {
|
||||
Token::ApostrophenatedWord(word)
|
||||
| Token::HyphenatedWord(word)
|
||||
| Token::Word(word) => {
|
||||
if needs_space {
|
||||
transformed_sentence.push_str(" ");
|
||||
needs_space = false;
|
||||
}
|
||||
|
||||
// Option<Polyphone>
|
||||
//pub type Polyphone = Vec<Phoneme>;
|
||||
//
|
||||
match arpabet.get_polyphone(&word) {
|
||||
None => {
|
||||
transformed_sentence.push_str(&word);
|
||||
needs_space = true;
|
||||
},
|
||||
Some(arpas) => {
|
||||
let collected : Vec<String>= arpas.iter()
|
||||
.map(|a| ["@", a.to_str()].join("")) // TODO: Add to arpabet library
|
||||
.collect();
|
||||
|
||||
let transformed = collected.join("");
|
||||
|
||||
transformed_sentence.push_str(&transformed);
|
||||
needs_space = true;
|
||||
},
|
||||
}
|
||||
},
|
||||
Token::Integer(integer_string) => {
|
||||
if needs_space {
|
||||
transformed_sentence.push_str(" ");
|
||||
needs_space = false;
|
||||
}
|
||||
// TODO
|
||||
/*let number_words = integer_string.parse::<i64>()
|
||||
.ok()
|
||||
.and_then(|number| numerics.convert_number(number).ok())
|
||||
.unwrap_or(Vec::new());*/
|
||||
|
||||
/*for word in number_words {
|
||||
if needs_space {
|
||||
encoded_buffer.push(space);
|
||||
needs_space = false;
|
||||
}
|
||||
if arpabet_lookup_and_encode_word(&word, &arpabet, &mut encoded_buffer) {
|
||||
needs_space = true;
|
||||
}
|
||||
}*/
|
||||
},
|
||||
Token::Punctuation(punctuation) => {
|
||||
match punctuation {
|
||||
Punctuation::Period => {
|
||||
transformed_sentence.push_str(".");
|
||||
needs_space = true;
|
||||
},
|
||||
Punctuation::Comma | Punctuation::Semicolon => {
|
||||
transformed_sentence.push_str(",");
|
||||
needs_space = true;
|
||||
},
|
||||
Punctuation::Question => {
|
||||
transformed_sentence.push_str("?");
|
||||
needs_space = true;
|
||||
},
|
||||
Punctuation::Exclamation => {
|
||||
transformed_sentence.push_str("!");
|
||||
needs_space = true;
|
||||
},
|
||||
Punctuation::Colon | Punctuation::Dash => {
|
||||
transformed_sentence.push_str(" - ");
|
||||
needs_space = false;
|
||||
},
|
||||
}
|
||||
},
|
||||
// TODO:
|
||||
Token::RealNumber(_) => continue,
|
||||
Token::CommaFormattedInteger(_) => continue,
|
||||
Token::CommaFormattedRealNumber(_) => continue,
|
||||
Token::Hashtag(_) => continue,
|
||||
Token::UsernameMention(_) => continue,
|
||||
Token::Url(_) => continue,
|
||||
Token::Unknown(_) => continue, // skip, we don't know what this is yet.
|
||||
}
|
||||
}
|
||||
|
||||
transformed_sentence
|
||||
}
|
||||
|
||||
/// Convert text to the encoding used in my Arpabet extension
|
||||
pub fn text_to_arpabet_encoding(arpabet: &Arpabet, text: &str) -> Vec<i64> {
|
||||
let sentence_tokenizer = SentenceTokenizer::new();
|
||||
|
||||
Reference in new issue
Block a user