utilities for text pipeline determination

Cargo.lock

@@ -4459,6 +4459,7 @@ name = "tts_common"
 version = "0.0.1"
 dependencies = [
  "anyhow",
+ "chrono",
  "container_common",
  "log",
  "once_cell",

crates/lib/tts_common/Cargo.toml

@@ -20,6 +20,7 @@ user_input_common = { path = "../user_input_common" }
 
 # External
 anyhow = "1.0.40"
+chrono = { version = "0.4.19", features = ["serde"] }
 log = "0.4.14"
 once_cell = "1.9.0"
 regex = "1.5.4"

crates/lib/tts_common/src/lib.rs

@@ -1,2 +1,4 @@
 pub mod clean_symbols;
-pub mod priority;
+pub mod priority;
+pub mod text_pipeline_defaults;
+pub mod text_pipelines;

crates/lib/tts_common/src/text_pipeline_defaults.rs

@@ -0,0 +1,57 @@
+use std::ops::Deref;
+use chrono::{DateTime, Utc};
+use once_cell::sync::Lazy;
+use crate::text_pipelines::{ENGLISH_V1, LEGACY_FAKEYOU};
+
+const ENGLISH_V1_EPOCH_STR : &'static str = "2022-05-01T00:00:00.00Z";
+
+/// Date we consider to switch models to "english_v1" instead of "legacy_fakeyou"
+static ENGLISH_V1_EPOCH : Lazy<DateTime<Utc>> = Lazy::new(|| {
+  let datetime = DateTime::parse_from_rfc3339(ENGLISH_V1_EPOCH_STR)
+      .expect("ENGLISH_V1_EPOCH must parse statically.");
+
+  datetime.with_timezone(&Utc)
+});
+
+/// Return a reasonable guess for default text pipeline when one isn't set.
+/// New models won't necessarily have a user-selected text pipeline (so we'll guess),
+/// and old models will need to be backfilled since we never collected that information.
+///
+///   maybe_model_created_at - `created_at` timestamp from database
+pub fn guess_text_pipeline_heuristic(maybe_model_created_at: Option<DateTime<Utc>>) -> &'static str {
+
+  if let Some(created_at) = maybe_model_created_at {
+    if created_at < *ENGLISH_V1_EPOCH {
+      return LEGACY_FAKEYOU;
+    }
+  }
+
+  ENGLISH_V1
+}
+
+#[cfg(test)]
+mod tests {
+  use chrono::{DateTime, Utc};
+  use crate::text_pipeline_defaults::guess_text_pipeline_heuristic;
+
+  #[test]
+  fn new_models_just_created_with_no_date_use_english_v1() {
+    assert_eq!(guess_text_pipeline_heuristic(None), "english_v1");
+  }
+
+  #[test]
+  fn newish_models_use_english_v1() {
+    let datetime = DateTime::parse_from_rfc3339("2022-07-01T00:00:00.00Z")
+        .expect("should parse")
+        .with_timezone(&Utc);
+    assert_eq!(guess_text_pipeline_heuristic(Some(datetime)), "english_v1");
+  }
+
+  #[test]
+  fn older_models_use_legacy_fakeyou() {
+    let datetime = DateTime::parse_from_rfc3339("2022-01-01T00:00:00.00Z")
+        .expect("should parse")
+        .with_timezone(&Utc);
+    assert_eq!(guess_text_pipeline_heuristic(Some(datetime)), "legacy_fakeyou");
+  }
+}

crates/lib/tts_common/src/text_pipelines.rs

@@ -0,0 +1,82 @@
+//! We support a variety of text pipeline systems.
+
+use std::collections::HashSet;
+use once_cell::sync::Lazy;
+
+// TODO: Introduce old vocodes models.
+//pub const LEGACY_VOCODES : &'static str = "legacy_vocodes";
+
+/// Introduction date: 2021.
+/// Legacy FakeYou models use graphemes by default. They "can" support arpabet segments manually
+/// specified by the user with curly brackets, but there is no guarantee that the model supports
+/// this. By default the model will be sent grapheme symbols converted to integers.
+pub const LEGACY_FAKEYOU : &'static str = "legacy_fakeyou";
+
+/// Introduction date: July 2022, though models trained earlier on arpabet should support this.
+/// Unlike "legacy_fakeyou", this forces arpabet lookup of graphemes in all cases possible. The
+/// integer encoding is the same, but there are some additional normalization routines.
+pub const ENGLISH_V1 : &'static str = "english_v1";
+
+/// Introduction date: approx July 2022.
+/// This was developed by Ezequiel and Mathias using a modified arpabet scheme similar to
+/// "english_v1".
+/// NB(2022-07-05): Technically not landed yet.
+pub const SPANISH_V1 : &'static str = "spanish_v1";
+
+/// Introduction date: approx July 2022.
+/// An improvement upon "spanish_v1" that uses an entirely different phonetic system.
+/// NB(2022-07-05): Technically not landed yet.
+pub const SPANISH_V2 : &'static str = "spanish_v2";
+
+static SUPPORTED_TEXT_PIPELINES : Lazy<HashSet<String>> = Lazy::new(|| {
+  let text_pipeline_names = [
+    LEGACY_FAKEYOU,
+    ENGLISH_V1,
+    SPANISH_V1,
+    SPANISH_V2,
+  ];
+  text_pipeline_names.iter()
+      .map(|tag| tag.to_string())
+      .collect::<HashSet<String>>()
+});
+
+/// Check if the text pipeline name (must be lowercase) is valid and supported.
+pub fn is_valid_text_pipeline(text_pipeline_name: &str) -> bool {
+  SUPPORTED_TEXT_PIPELINES.contains(text_pipeline_name)
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::text_pipelines::is_valid_text_pipeline;
+
+  #[test]
+  fn valid_text_pipelines() {
+    assert!(is_valid_text_pipeline("legacy_fakeyou"));
+    assert!(is_valid_text_pipeline("english_v1"));
+    assert!(is_valid_text_pipeline("spanish_v1"));
+    assert!(is_valid_text_pipeline("spanish_v2"));
+  }
+
+  #[test]
+  fn invalid_text_pipelines() {
+    // Garbage
+    assert!(!is_valid_text_pipeline(""));
+    assert!(!is_valid_text_pipeline("asdf"));
+
+    // NB: Must be lower case
+    assert!(!is_valid_text_pipeline("LEGACY_FAKEYOU"));
+    assert!(!is_valid_text_pipeline("ENGLISH_V1"));
+    assert!(!is_valid_text_pipeline("SPANISH_V1"));
+    assert!(!is_valid_text_pipeline("SPANISH_V2"));
+
+    // NB: Not yet supported
+    assert!(!is_valid_text_pipeline("legacy_vocodes"));
+    assert!(!is_valid_text_pipeline("spanish_v3"));
+    assert!(!is_valid_text_pipeline("english_v2"));
+
+    // Wrong names
+    assert!(!is_valid_text_pipeline("english"));
+    assert!(!is_valid_text_pipeline("spanish"));
+    assert!(!is_valid_text_pipeline("vocodes"));
+  }
+}