From 60211d6df69aa265e195b510e9c53eafeb022697 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Tue, 23 Jun 2020 02:52:11 -0400
Subject: [PATCH] analytic /words endpoint

---
 tts_service/src/endpoints/mod.rs   |  3 +-
 tts_service/src/endpoints/words.rs | 77 ++++++++++++++++++++++++++++++
 tts_service/src/main.rs            |  2 +
 3 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 tts_service/src/endpoints/words.rs

diff --git a/tts_service/src/endpoints/mod.rs b/tts_service/src/endpoints/mod.rs
index f499c9fdc5..6b53766332 100644
--- a/tts_service/src/endpoints/mod.rs
+++ b/tts_service/src/endpoints/mod.rs
@@ -7,4 +7,5 @@ pub mod sentences;
 pub mod speak;
 pub mod speak_with_spectrogram;
 pub mod speakers;
-pub mod tts;
\ No newline at end of file
+pub mod tts;
+pub mod words;
\ No newline at end of file
diff --git a/tts_service/src/endpoints/words.rs b/tts_service/src/endpoints/words.rs
new file mode 100644
index 0000000000..ffa2cfda65
--- /dev/null
+++ b/tts_service/src/endpoints/words.rs
@@ -0,0 +1,77 @@
+use actix_web::web::{Data, Json, Query};
+use actix_web::http::StatusCode;
+use actix_web::{HttpRequest, get, HttpResponse, Either};
+
+use std::sync::Arc;
+use crate::AppState;
+use crate::database::model::Sentence;
+use std::collections::HashMap;
+
+/// Paginated responses in the format expected by Tabulator.js
+#[derive(Serialize, Debug)]
+pub struct WordsResult {
+  results: Vec<(String, i64)>,
+}
+
+#[get("/words")]
+pub async fn get_words(
+  _request: HttpRequest,
+  app_state: Data<Arc<AppState>>
+) -> Either<Json<WordsResult>, std::io::Result<HttpResponse>>{
+
+  let app_state = app_state.into_inner();
+
+  // Major performance implications, but this is only going to be called occasionally
+  let limit = 20_000_000;
+  let offset = 0;
+  let sort_ascending = true;
+
+  let records = match Sentence::load(&app_state.database_connector, limit, offset, sort_ascending) {
+    Err(e) => {
+      error!("Couldn't query database for sentences: {:?}", e);
+      return Either::B(Ok(HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR)
+          .content_type("text/plain")
+          .body("Couldn't query database database")));
+    },
+    Ok(results) => results,
+  };
+
+  let sentences : Vec<&str> = records
+      .iter()
+      .map(|r| r.sentence.as_ref())
+      .collect();
+
+  let words : Vec<String> = sentences
+      .iter()
+      .flat_map(|s| s.split_whitespace())
+      .map(|s| s.trim())
+      .map(|s| s.to_lowercase())
+      .map(|s| s.replace("!", ""))
+      .map(|s| s.replace(",", ""))
+      .map(|s| s.replace(".", ""))
+      .map(|s| s.replace("?", ""))
+      .collect();
+
+  let mut word_counts : HashMap<String, i64> = HashMap::new();
+
+  for word in words {
+    if(!word_counts.contains_key(&word)) {
+      word_counts.insert(word.clone(), 0);
+    }
+    word_counts.get_mut(&word).map(|count| *count += 1);
+  }
+
+  let mut word_counts_sortable : Vec<(String, i64)> = Vec::with_capacity(word_counts.len());
+
+  for (word, count) in word_counts {
+    word_counts_sortable.push((word.to_string(), count));
+  }
+
+  // Sorts descending by count
+  word_counts_sortable.sort_by(|a, b| b.1.cmp(&a.1));
+
+
+  Either::A(Json(WordsResult {
+    results: word_counts_sortable
+  }))
+}
diff --git a/tts_service/src/main.rs b/tts_service/src/main.rs
index 8baa210872..0f3e8745c2 100644
--- a/tts_service/src/main.rs
+++ b/tts_service/src/main.rs
@@ -43,6 +43,7 @@ use crate::model::model_cache::ModelCache;
 use crate::model::model_config::ModelConfigs;
 use crate::text::checker::TextChecker;
 use crate::endpoints::speak_with_spectrogram::post_speak_with_spectrogram;
+use crate::endpoints::words::get_words;
 
 const ENV_ASSET_DIRECTORY: &'static str = "ASSET_DIRECTORY";
 const ENV_BIND_ADDRESS: &'static str = "BIND_ADDRESS";
@@ -246,6 +247,7 @@ async fn run_server(app_state: AppState, server_args: ServerArgs) -> std::io::Re
       .service(get_models)
       .service(get_speakers)
       .service(get_sentences)
+      .service(get_words)
       .app_data(arc.clone())
     )
     .bind(bind_address)?