query testing harness

crates/cli/production/elasticsearch-cli/src/cli_args.rs

@@ -30,6 +30,9 @@ pub enum Action {
   /// Search the model weights
   SearchModelWeights,
 
+  /// Evaluate the search algorithm
+  EvaluateModelWeights,
+
   /// Reindex media_files into ElasticSearch
   /// For now, this is just media files that are relevant to the engine.
   ReindexMediaFiles,

crates/cli/production/elasticsearch-cli/src/main.rs

@@ -24,6 +24,7 @@ use crate::plans::create_all_tts_documents::create_all_tts_documents;
 use crate::plans::media_files::create_dimensional_media_file_documents::create_dimensional_media_file_documents;
 use crate::plans::media_files::test_search_media_files_documents::test_search_media_files;
 use crate::plans::model_weights::create_all_model_weight_documents::create_all_model_weight_documents;
+use crate::plans::model_weights::evaluate::evaluate_model_weights_search::evaluate_model_weights_search;
 use crate::plans::model_weights::test_search_model_weights_documents::test_search_model_weights_documents;
 
 pub mod cli_args;
@@ -33,7 +34,7 @@ pub mod plans;
 pub async fn main() -> AnyhowResult<()> {
   println!("elasticsearch-cli: operational tooling for Elasticsearch");
 
-  easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
+  easyenv::init_all_with_default_logging(Some("info"));
 
   // NB: This secrets file differs from the rest because we might actually want to cross
   // development/production boundaries for seeding, or scope to production for rebuilding indices.
@@ -67,6 +68,10 @@ pub async fn main() -> AnyhowResult<()> {
         println!("Result: {:#?}", result);
       }
     }
+    Action::EvaluateModelWeights => {
+      info!("Evaluate model weights search...");
+      let _ = evaluate_model_weights_search(&elasticsearch).await?;
+    }
     Action::ReindexMediaFiles => {
       info!("Reindexing media files...");
       create_dimensional_media_file_documents(&mysql, &elasticsearch).await?;

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/asserts.rs

@@ -0,0 +1,44 @@
+use std::collections::HashSet;
+use elasticsearch::Elasticsearch;
+use log::error;
+use errors::{anyhow, AnyhowResult};
+use crate::plans::model_weights::evaluate::search::search_term_only;
+use crate::plans::model_weights::evaluate::titles::{print_titles, to_title_set};
+
+pub async fn assert_search_term_contains_titles(
+  client: &Elasticsearch,
+  search_term: &str,
+  expected_titles: &Vec<&str>,
+) -> AnyhowResult<()> {
+  let results = search_term_only(search_term, client).await?;
+  let titles = to_title_set(&results);
+
+  println!("Titles:");
+  print_titles(&results);
+
+  if let Err(err) = assert_contains_all(&titles, expected_titles, search_term) {
+    error!("Title not found for search: {}", search_term);
+    print_titles(&results);
+    return Err(err);
+  }
+
+  Ok(())
+}
+pub fn assert_contains(titles: &HashSet<String>, expected: &str, search_term: &str) -> AnyhowResult<()> {
+  if !titles.contains(expected) {
+    error!("Expected title not found: {} for search {} ({} results)",
+      expected, search_term, titles.len());
+
+    return Err(anyhow!("Expected title not found: {} for search {} ({} results)",
+      expected, search_term, titles.len()));
+  }
+  Ok(())
+}
+
+pub fn assert_contains_all(titles: &HashSet<String>, expected: &Vec<&str>, search_term: &str) -> AnyhowResult<()> {
+  for title in expected.iter() {
+    assert_contains(titles, title, search_term)?;
+  }
+  Ok(())
+}
+

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/evaluate_model_weights_search.rs

@@ -0,0 +1,10 @@
+use crate::plans::model_weights::evaluate::test_cases::zelda::test_zelda_searches;
+use elasticsearch::Elasticsearch;
+use errors::AnyhowResult;
+
+pub async fn evaluate_model_weights_search(client: &Elasticsearch) -> AnyhowResult<()> {
+  test_zelda_searches(client).await?;
+
+  Ok(())
+}
+

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/mod.rs

@@ -0,0 +1,5 @@
+pub mod asserts;
+pub mod evaluate_model_weights_search;
+pub mod search;
+pub mod test_cases;
+pub mod titles;

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/search.rs

@@ -0,0 +1,19 @@
+use elasticsearch::Elasticsearch;
+use elasticsearch_schema::documents::model_weight_document::ModelWeightDocument;
+use elasticsearch_schema::searches::search_model_weights::search_model_weights::{search_model_weights, SearchArgs};
+use errors::AnyhowResult;
+
+pub async fn search_term_only(search_term: &str, client: &Elasticsearch) -> AnyhowResult<Vec<ModelWeightDocument>> {
+  search_model_weights(SearchArgs {
+    search_term,
+    maybe_creator_user_token: None,
+    maybe_ietf_primary_language_subtag: None,
+    maybe_weights_categories: None,
+    maybe_weights_types: None,
+    sort_field: None,
+    sort_direction: None,
+    minimum_score: None,
+    client: &client,
+  }).await
+}
+

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/mod.rs

@@ -0,0 +1 @@
+pub mod zelda;

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/zelda.rs

@@ -0,0 +1,31 @@
+use crate::plans::model_weights::evaluate::asserts::assert_search_term_contains_titles;
+use elasticsearch::Elasticsearch;
+use errors::AnyhowResult;
+
+pub async fn test_zelda_searches(
+  client: &Elasticsearch
+) -> AnyhowResult<()> {
+
+  assert_search_term_contains_titles(client, "zeld", &vec![
+    "Link (Zelda CDI) (Français)",
+    "Zelda (BOTW Español Latino)",
+    "Zelda (BOTW) (Ita)",
+    "Zelda (BOTW)",
+    "Zelda (Breath of the Wild)",
+    "Zelda Rubinstein",
+    "Zelda. (The Legend of Zelda: Breath of the Wild, Castillian Spanish.) Versión MEJORADA.",
+  ]).await?;
+
+  assert_search_term_contains_titles(client, "zelad", &vec![
+    "Link (Zelda CDI) (Français)",
+    "Zelda (BOTW Español Latino)",
+    "Zelda (BOTW) (Ita)",
+    "Zelda (BOTW)",
+    "Zelda (Breath of the Wild)",
+    "Zelda Rubinstein",
+    "Zelda. (The Legend of Zelda: Breath of the Wild, Castillian Spanish.) Versión MEJORADA.",
+  ]).await?;
+
+  Ok(())
+}
+

crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/titles.rs

@@ -0,0 +1,13 @@
+use std::collections::HashSet;
+use elasticsearch_schema::documents::model_weight_document::ModelWeightDocument;
+
+pub fn print_titles(results: &Vec<ModelWeightDocument>) {
+  for result in results {
+    println!("  - result: {:#?} ({:?})", result.title, result.token);
+  }
+}
+
+pub fn to_title_set(results: &Vec<ModelWeightDocument>) -> HashSet<String> {
+  results.iter().map(|result| result.title.clone()).collect()
+}
+

crates/cli/production/elasticsearch-cli/src/plans/model_weights/mod.rs

@@ -1,2 +1,3 @@
-pub mod test_search_model_weights_documents;
-pub mod create_all_model_weight_documents;
+pub mod create_all_model_weight_documents;
+pub mod evaluate;
+pub mod test_search_model_weights_documents;

crates/lib/elasticsearch_schema/src/searches/search_model_weights/min_score.rs

@@ -21,12 +21,12 @@ pub fn add_min_score(
   }
 
   let minimum_score = match search_term.len() {
-    1..=3 => 1,
-    4 => 5,
-    5 => 10,
-    6 => 30,
-    7 => 50,
-    _ => 100,
+    1..=3 => 0,
+    4 => 1,
+    5 => 3,
+    6 => 5,
+    7 => 7,
+    _ => 10,
   };
 
   if let Some(mut object) = query.as_object_mut() {

crates/my-workspace-hack/Cargo.toml

@@ -4,6 +4,7 @@
 
 [package]
 name = "my-workspace-hack"
+edition = "2018"
 version = "0.1.0"
 description = "workspace-hack package, managed by hakari"
 # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.