mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-19 08:05:20 +00:00
Added phoneme padding params and pause synthesis
This commit is contained in:
1 parent
489aff2540
commit
159eff2a1e
7 files changed
+231
-31
No files matched your search
@@ -1,5 +1,6 @@
|
||||
// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
|
||||
|
||||
pub mod pause;
|
||||
pub mod speed;
|
||||
pub mod volume;
|
||||
|
||||
@@ -37,9 +37,12 @@ const SPEED_PARAM : &'static str = "spd";
|
||||
const USE_PHONEMES_PARAM: &'static str = "up";
|
||||
const USE_WORDS_PARAM: &'static str = "uw";
|
||||
const VOLUME_PARAM : &'static str = "vol";
|
||||
const MONOPHONE_PADDING_START_PARAM : &'static str = "mps";
|
||||
const MONOPHONE_PADDING_END_PARAM : &'static str = "mpe";
|
||||
const POLYPHONE_PADDING_END_PARAM : &'static str = "ppe";
|
||||
|
||||
/// Represents a request to this endpoint.
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
struct SpeakRequest {
|
||||
/** The sentence to be spoken. */
|
||||
pub sentence: String,
|
||||
@@ -58,6 +61,15 @@ struct SpeakRequest {
|
||||
|
||||
/** Whether to use words. */
|
||||
pub use_words: bool,
|
||||
|
||||
/** Padding before a monophone. */
|
||||
pub monophone_padding_start: Option<u16>,
|
||||
|
||||
/** Padding after a monophone. */
|
||||
pub monophone_padding_end: Option<u16>,
|
||||
|
||||
/** Padding after a polyphone. */
|
||||
pub polyphone_padding_end: Option<u16>,
|
||||
}
|
||||
|
||||
enum SpeakerRequestError {
|
||||
@@ -82,13 +94,14 @@ pub struct AudioSynthHandler {
|
||||
}
|
||||
|
||||
impl SpeakRequest {
|
||||
pub fn parse(http_request: &mut Request) -> Result<SpeakRequest, SpeakerRequestError> {
|
||||
pub fn parse(http_request: &mut Request)
|
||||
-> Result<SpeakRequest, SpeakerRequestError> {
|
||||
|
||||
let sentence_error = Err(SpeakerRequestError::SentenceInvalid);
|
||||
let speaker_error = Err(SpeakerRequestError::SpeakerInvalid);
|
||||
|
||||
// Get the request sentence and speaker.
|
||||
// TODO: Cleanup
|
||||
// TODO: OMFG WTF CLEANUP THIS GARBAGE.
|
||||
match http_request.get_ref::<UrlEncodedQuery>() {
|
||||
Err(_) => { return sentence_error; },
|
||||
Ok(ref map) => {
|
||||
@@ -179,6 +192,69 @@ impl SpeakRequest {
|
||||
},
|
||||
};
|
||||
|
||||
let mps : Option<u16> = match map.get(MONOPHONE_PADDING_START_PARAM) {
|
||||
None => { None },
|
||||
Some(list) => {
|
||||
match list.get(0) {
|
||||
None => { None },
|
||||
Some(s) => {
|
||||
match s.trim().parse::<u16>() {
|
||||
Err(_) => None,
|
||||
Ok(i) => {
|
||||
if i == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let mpe : Option<u16> = match map.get(MONOPHONE_PADDING_END_PARAM) {
|
||||
None => { None },
|
||||
Some(list) => {
|
||||
match list.get(0) {
|
||||
None => { None },
|
||||
Some(s) => {
|
||||
match s.trim().parse::<u16>() {
|
||||
Err(_) => None,
|
||||
Ok(i) => {
|
||||
if i == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let ppe : Option<u16> = match map.get(POLYPHONE_PADDING_END_PARAM) {
|
||||
None => { None },
|
||||
Some(list) => {
|
||||
match list.get(0) {
|
||||
None => { None },
|
||||
Some(s) => {
|
||||
match s.trim().parse::<u16>() {
|
||||
Err(_) => None,
|
||||
Ok(i) => {
|
||||
if i == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
Ok(SpeakRequest {
|
||||
sentence: sen,
|
||||
speaker: spk,
|
||||
@@ -186,6 +262,9 @@ impl SpeakRequest {
|
||||
speed: speed,
|
||||
use_phonemes: use_phonemes,
|
||||
use_words: use_words,
|
||||
monophone_padding_start: mps,
|
||||
monophone_padding_end: mpe,
|
||||
polyphone_padding_end: ppe,
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -236,9 +315,7 @@ impl Handler for AudioSynthHandler {
|
||||
}
|
||||
}
|
||||
|
||||
let result = self.create_audio(&request.speaker, &request.sentence,
|
||||
request.use_words, request.use_phonemes,
|
||||
request.volume, request.speed);
|
||||
let result = self.create_audio(request);
|
||||
|
||||
let mime_type = "audio/wav".parse::<Mime>().unwrap();
|
||||
|
||||
@@ -261,14 +338,19 @@ impl AudioSynthHandler {
|
||||
|
||||
// TODO: Return errors.
|
||||
/// Create audio from the sentence.
|
||||
fn create_audio(&self, speaker: &str, sentence: &str, use_words: bool,
|
||||
use_phonemes: bool, volume: Option<f32>,
|
||||
speed: Option<f32>) -> Vec<u8> {
|
||||
fn create_audio(&self, request: SpeakRequest) -> Vec<u8> {
|
||||
match self.synthesizer.read() {
|
||||
Err(_) => Vec::new(), // TODO Actual error.
|
||||
Ok(synth) => {
|
||||
let generated = synth.generate(sentence, speaker, use_words,
|
||||
use_phonemes, volume, speed);
|
||||
let generated = synth.generate(&request.sentence,
|
||||
&request.speaker,
|
||||
request.use_words,
|
||||
request.use_phonemes,
|
||||
request.volume,
|
||||
request.speed,
|
||||
request.monophone_padding_start,
|
||||
request.monophone_padding_end,
|
||||
request.polyphone_padding_end);
|
||||
match generated {
|
||||
Err(e) => {
|
||||
println!("Error synthesizing: {:?}", e);
|
||||
|
||||
+38
-11
@@ -2,6 +2,7 @@
|
||||
|
||||
use arpabet::ArpabetDictionary;
|
||||
use audiobank::Audiobank;
|
||||
use effects::pause::generate_pause;
|
||||
use effects::speed::change_speed;
|
||||
use effects::volume::change_volume;
|
||||
use error::SynthError;
|
||||
@@ -41,10 +42,17 @@ impl Synthesizer {
|
||||
/**
|
||||
* Generate a spoken wav from text input.
|
||||
*/
|
||||
pub fn generate(&self, sentence: &str, speaker: &str,
|
||||
use_words: bool, use_phonemes: bool,
|
||||
volume: Option<f32>, speed: Option<f32>)
|
||||
-> Result<WavBytes, SynthError> {
|
||||
pub fn generate(&self,
|
||||
sentence: &str,
|
||||
speaker: &str,
|
||||
use_words: bool,
|
||||
use_phonemes: bool,
|
||||
volume: Option<f32>,
|
||||
speed: Option<f32>,
|
||||
monophone_padding_start: Option<u16>,
|
||||
monophone_padding_end: Option<u16>,
|
||||
polyphone_padding_end: Option<u16>)
|
||||
-> Result<WavBytes, SynthError> {
|
||||
|
||||
let mut words = split_sentence(sentence);
|
||||
|
||||
@@ -71,7 +79,12 @@ impl Synthesizer {
|
||||
}
|
||||
|
||||
if !word_added && use_phonemes {
|
||||
word_added = self.concatenate_polyphone(&mut concatenated_waveform, speaker, word);
|
||||
word_added = self.concatenate_polyphone(&mut concatenated_waveform,
|
||||
speaker,
|
||||
word,
|
||||
monophone_padding_start,
|
||||
monophone_padding_end,
|
||||
polyphone_padding_end);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,9 +128,13 @@ impl Synthesizer {
|
||||
/// Concatenate a polyphone corresponding to the word to the
|
||||
/// waveform we're building. Returns whether or not the word was
|
||||
/// successfully found and concatenated.
|
||||
fn concatenate_polyphone(&self, concatenated_waveform: &mut Vec<i16>,
|
||||
speaker: &str, word: &str) -> bool {
|
||||
|
||||
fn concatenate_polyphone(&self,
|
||||
concatenated_waveform: &mut Vec<i16>,
|
||||
speaker: &str,
|
||||
word: &str,
|
||||
monophone_padding_start: Option<u16>,
|
||||
monophone_padding_end: Option<u16>,
|
||||
polyphone_padding_end: Option<u16>) -> bool {
|
||||
|
||||
let polyphone = match self.arpabet_dictionary.get_polyphone(word) {
|
||||
Some(p) => { p },
|
||||
@@ -157,15 +174,25 @@ impl Synthesizer {
|
||||
continue;
|
||||
},
|
||||
Some(waveform_data) => {
|
||||
if monophone_padding_start.is_some() {
|
||||
let pause = generate_pause(monophone_padding_start.unwrap());
|
||||
concatenated_waveform.extend(pause);
|
||||
}
|
||||
|
||||
concatenated_waveform.extend(waveform_data);
|
||||
|
||||
if monophone_padding_end.is_some() {
|
||||
let pause = generate_pause(monophone_padding_end.unwrap());
|
||||
concatenated_waveform.extend(pause);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Insert space after polyphone.
|
||||
match self.audiobank.get_misc("pause") {
|
||||
None => {},
|
||||
Some(pause) => { concatenated_waveform.extend(pause); },
|
||||
if polyphone_padding_end.is_some() {
|
||||
let pause = generate_pause(polyphone_padding_end.unwrap());
|
||||
concatenated_waveform.extend(pause);
|
||||
}
|
||||
|
||||
true
|
||||
|
||||
@@ -40,17 +40,23 @@
|
||||
speaker = $('select').val(),
|
||||
volume = $('input#vol').val(),
|
||||
speed = $('input#speed').val(),
|
||||
mps = $('input#monophone_padding_start').val(),
|
||||
mpe = $('input#monophone_padding_end').val(),
|
||||
ppe = $('input#polyphone_padding_end').val(),
|
||||
query = encodeURIComponent(sentence),
|
||||
url = '/speak?v=' + speaker
|
||||
+ '&s=' + query
|
||||
+ '&vol=' + volume
|
||||
+ '&spd=' + speed
|
||||
+ '&mps=' + mps
|
||||
+ '&mpe=' + mpe
|
||||
+ '&ppe=' + ppe
|
||||
+ '&up=' + $('#use_phonemes').prop('checked')
|
||||
+ '&uw=' + $('#use_words').prop('checked');
|
||||
|
||||
ev.preventDefault();
|
||||
|
||||
Url.setState(speaker, sentence, volume, speed);
|
||||
Url.setState(speaker, sentence, volume, speed, mps, mpe, ppe);
|
||||
Sound.play(url);
|
||||
|
||||
return false;
|
||||
|
||||
@@ -17,6 +17,9 @@
|
||||
speaker = Url.getSpeaker() || Library.getDefaultSpeaker(),
|
||||
volume = Url.getVolume() || Volume.DEFAULT,
|
||||
speed = Url.getSpeed() || 1.0,
|
||||
mps = Url.getMonophonePaddingStart() || 0,
|
||||
mpe = Url.getMonophonePaddingEnd() || 0,
|
||||
ppe = Url.getPolyphonePaddingEnd() || 600,
|
||||
urlPreviousState = false;
|
||||
|
||||
if (speaker) {
|
||||
@@ -28,12 +31,17 @@
|
||||
urlPreviousState = true;
|
||||
}
|
||||
|
||||
$('input#vol').val(volume);
|
||||
speed = parseFloat(speed).toFixed(2);
|
||||
$('input#speed').val(speed);
|
||||
$('input#vol').val(volume);
|
||||
|
||||
$('#use_phonemes').prop('use_phonemes', Url.getUsePhonemes());
|
||||
$('#use_words').prop('use_words', Url.getUseWords());
|
||||
|
||||
$('input#monophone_padding_start').val(mps);
|
||||
$('input#monophone_padding_end').val(mpe);
|
||||
$('input#polyphone_padding_end').val(ppe);
|
||||
|
||||
Library.load(function(library) { Ui.libraryLoadCallback(library); });
|
||||
Sound.install();
|
||||
|
||||
|
||||
+61
-3
@@ -22,6 +22,10 @@
|
||||
/** JSON key for the `use_words` param. */
|
||||
_USE_WORDS_KEY: 'uw',
|
||||
|
||||
_MONOPHONE_PADDING_START_KEY: 'mps',
|
||||
_MONOPHONE_PADDING_END_KEY: 'mpe',
|
||||
_POLYPHONE_PADDING_END_KEY: 'ppe',
|
||||
|
||||
/** Get the sentence from the `window.location`. */
|
||||
getSentence: function() {
|
||||
var state = this.parseState(window.location),
|
||||
@@ -84,16 +88,67 @@
|
||||
}
|
||||
},
|
||||
|
||||
/** Get the MPS value from the `window.location`. */
|
||||
getMonophonePaddingStart: function() {
|
||||
var state = this.parseState(window.location);
|
||||
if (!state || !(this._MONOPHONE_PADDING_START_KEY in state)) {
|
||||
return null;
|
||||
} else {
|
||||
return state[this._MONOPHONE_PADDING_START_KEY];
|
||||
}
|
||||
},
|
||||
|
||||
/** Get the MPE value from the `window.location`. */
|
||||
getMonophonePaddingEnd: function() {
|
||||
var state = this.parseState(window.location);
|
||||
if (!state || !(this._MONOPHONE_PADDING_END_KEY in state)) {
|
||||
return null;
|
||||
} else {
|
||||
return state[this._MONOPHONE_PADDING_END_KEY];
|
||||
}
|
||||
},
|
||||
|
||||
/** Get the PPE value from the `window.location`. */
|
||||
getPolyphonePaddingEnd: function() {
|
||||
var state = this.parseState(window.location);
|
||||
if (!state || !(this._POLYPHONE_PADDING_END_KEY in state)) {
|
||||
return null;
|
||||
} else {
|
||||
return state[this._POLYPHONE_PADDING_END_KEY];
|
||||
}
|
||||
},
|
||||
|
||||
// TODO: Independent functions to set both speaker and sentence separately.
|
||||
/** Set the `window.history` speaker, sentence, phoneme use, and word use. */
|
||||
setState: function(speaker, rawSentence, volume, speed, usePhonemes, useWords) {
|
||||
var urlHash = this.fromParams(speaker, rawSentence, volume, speed, usePhonemes, useWords);
|
||||
setState: function(speaker,
|
||||
rawSentence,
|
||||
volume,
|
||||
speed,
|
||||
usePhonemes,
|
||||
useWords,
|
||||
monophonePaddingStart,
|
||||
monophonePaddingEnd,
|
||||
polyphonePaddingEnd) {
|
||||
|
||||
var urlHash = this.fromParams(speaker, rawSentence, volume, speed,
|
||||
usePhonemes, useWords, monophonePaddingStart, monophonePaddingEnd,
|
||||
polyphonePaddingEnd);
|
||||
|
||||
console.log('set state, urlhash = ', urlHash);
|
||||
window.history.replaceState(null, null, urlHash);
|
||||
},
|
||||
|
||||
/** Encode a speaker and sentence in a URL hash. */
|
||||
fromParams: function(speaker, rawSentence, volume, speed, usePhonemes, useWords) {
|
||||
fromParams: function(speaker,
|
||||
rawSentence,
|
||||
volume,
|
||||
speed,
|
||||
usePhonemes,
|
||||
useWords,
|
||||
monophonePaddingStart,
|
||||
monophonePaddingEnd,
|
||||
polyphonePaddingEnd) {
|
||||
|
||||
var cleanedSentence = SentenceHelper.cleanSentence(rawSentence),
|
||||
cleanedSpeaker = speaker, // TODO: Check against speakers.
|
||||
cleanedVolume = volume, // TODO: Filter invalid values.
|
||||
@@ -107,6 +162,9 @@
|
||||
state[this._SPEED_KEY] = speed; // TODO: Filter invalid values.
|
||||
state[this._USE_PHONEMES_KEY] = !!usePhonemes;
|
||||
state[this._USE_WORDS_KEY] = !!useWords;
|
||||
state[this._MONOPHONE_PADDING_START] = monophonePaddingStart;
|
||||
state[this._MONOPHONE_PADDING_END] = monophonePaddingEnd;
|
||||
state[this._POLYPHONE_PADDING_END] = polyphonePaddingEnd;
|
||||
|
||||
json = JSON.stringify(state);
|
||||
|
||||
|
||||
+22
-4
@@ -26,14 +26,32 @@
|
||||
<div id="options">
|
||||
<input type="checkbox" id="use_words" name="use_words"
|
||||
checked="checked">
|
||||
<label for="use_words">Words</label>
|
||||
<label for="use_words">Use Words</label>
|
||||
<input type="checkbox" id="use_phonemes" name="use_phonemes"
|
||||
checked="checked">
|
||||
<label for="use_phonemes">Phonemes</label>
|
||||
<label for="use_phonemes">Use Phonemes</label>
|
||||
</div>
|
||||
|
||||
<div id="more_options">
|
||||
<input id="speed" value="">
|
||||
<label for="speed">Speed</label>
|
||||
<div>
|
||||
<input id="speed" value="">
|
||||
<label for="speed">Speed</label>
|
||||
</div>
|
||||
<div>
|
||||
<input id="monophone_padding_start" value="">
|
||||
<label for="monophone_padding_start">
|
||||
Monophone Padding Start</label>
|
||||
</div>
|
||||
<div>
|
||||
<input id="monophone_padding_end" value="">
|
||||
<label for="monophone_padding_end">
|
||||
Monophone Padding End</label>
|
||||
</div>
|
||||
<div>
|
||||
<input id="polyphone_padding_end" value="">
|
||||
<label for="polyphone_padding_end">
|
||||
Polyphone Padding End</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</form>
|
||||
|
||||
Reference in new issue
Block a user