diff --git a/crates/cli/production/elasticsearch-cli/src/cli_args.rs b/crates/cli/production/elasticsearch-cli/src/cli_args.rs index cd631f434c..b4f1f935f3 100644 --- a/crates/cli/production/elasticsearch-cli/src/cli_args.rs +++ b/crates/cli/production/elasticsearch-cli/src/cli_args.rs @@ -30,6 +30,9 @@ pub enum Action { /// Search the model weights SearchModelWeights, + /// Evaluate the search algorithm + EvaluateModelWeights, + /// Reindex media_files into ElasticSearch /// For now, this is just media files that are relevant to the engine. ReindexMediaFiles, diff --git a/crates/cli/production/elasticsearch-cli/src/main.rs b/crates/cli/production/elasticsearch-cli/src/main.rs index cc7d862852..29d30b3709 100644 --- a/crates/cli/production/elasticsearch-cli/src/main.rs +++ b/crates/cli/production/elasticsearch-cli/src/main.rs @@ -24,6 +24,7 @@ use crate::plans::create_all_tts_documents::create_all_tts_documents; use crate::plans::media_files::create_dimensional_media_file_documents::create_dimensional_media_file_documents; use crate::plans::media_files::test_search_media_files_documents::test_search_media_files; use crate::plans::model_weights::create_all_model_weight_documents::create_all_model_weight_documents; +use crate::plans::model_weights::evaluate::evaluate_model_weights_search::evaluate_model_weights_search; use crate::plans::model_weights::test_search_model_weights_documents::test_search_model_weights_documents; pub mod cli_args; @@ -33,7 +34,7 @@ pub mod plans; pub async fn main() -> AnyhowResult<()> { println!("elasticsearch-cli: operational tooling for Elasticsearch"); - easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + easyenv::init_all_with_default_logging(Some("info")); // NB: This secrets file differs from the rest because we might actually want to cross // development/production boundaries for seeding, or scope to production for rebuilding indices. @@ -67,6 +68,10 @@ pub async fn main() -> AnyhowResult<()> { println!("Result: {:#?}", result); } } + Action::EvaluateModelWeights => { + info!("Evaluate model weights search..."); + let _ = evaluate_model_weights_search(&elasticsearch).await?; + } Action::ReindexMediaFiles => { info!("Reindexing media files..."); create_dimensional_media_file_documents(&mysql, &elasticsearch).await?; diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/asserts.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/asserts.rs new file mode 100644 index 0000000000..6eb39df53c --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/asserts.rs @@ -0,0 +1,44 @@ +use std::collections::HashSet; +use elasticsearch::Elasticsearch; +use log::error; +use errors::{anyhow, AnyhowResult}; +use crate::plans::model_weights::evaluate::search::search_term_only; +use crate::plans::model_weights::evaluate::titles::{print_titles, to_title_set}; + +pub async fn assert_search_term_contains_titles( + client: &Elasticsearch, + search_term: &str, + expected_titles: &Vec<&str>, +) -> AnyhowResult<()> { + let results = search_term_only(search_term, client).await?; + let titles = to_title_set(&results); + + println!("Titles:"); + print_titles(&results); + + if let Err(err) = assert_contains_all(&titles, expected_titles, search_term) { + error!("Title not found for search: {}", search_term); + print_titles(&results); + return Err(err); + } + + Ok(()) +} +pub fn assert_contains(titles: &HashSet, expected: &str, search_term: &str) -> AnyhowResult<()> { + if !titles.contains(expected) { + error!("Expected title not found: {} for search {} ({} results)", + expected, search_term, titles.len()); + + return Err(anyhow!("Expected title not found: {} for search {} ({} results)", + expected, search_term, titles.len())); + } + Ok(()) +} + +pub fn assert_contains_all(titles: &HashSet, expected: &Vec<&str>, search_term: &str) -> AnyhowResult<()> { + for title in expected.iter() { + assert_contains(titles, title, search_term)?; + } + Ok(()) +} + diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/evaluate_model_weights_search.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/evaluate_model_weights_search.rs new file mode 100644 index 0000000000..f8986f58c5 --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/evaluate_model_weights_search.rs @@ -0,0 +1,10 @@ +use crate::plans::model_weights::evaluate::test_cases::zelda::test_zelda_searches; +use elasticsearch::Elasticsearch; +use errors::AnyhowResult; + +pub async fn evaluate_model_weights_search(client: &Elasticsearch) -> AnyhowResult<()> { + test_zelda_searches(client).await?; + + Ok(()) +} + diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/mod.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/mod.rs new file mode 100644 index 0000000000..6a0858af45 --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/mod.rs @@ -0,0 +1,5 @@ +pub mod asserts; +pub mod evaluate_model_weights_search; +pub mod search; +pub mod test_cases; +pub mod titles; \ No newline at end of file diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/search.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/search.rs new file mode 100644 index 0000000000..d0895f5607 --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/search.rs @@ -0,0 +1,19 @@ +use elasticsearch::Elasticsearch; +use elasticsearch_schema::documents::model_weight_document::ModelWeightDocument; +use elasticsearch_schema::searches::search_model_weights::search_model_weights::{search_model_weights, SearchArgs}; +use errors::AnyhowResult; + +pub async fn search_term_only(search_term: &str, client: &Elasticsearch) -> AnyhowResult> { + search_model_weights(SearchArgs { + search_term, + maybe_creator_user_token: None, + maybe_ietf_primary_language_subtag: None, + maybe_weights_categories: None, + maybe_weights_types: None, + sort_field: None, + sort_direction: None, + minimum_score: None, + client: &client, + }).await +} + diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/mod.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/mod.rs new file mode 100644 index 0000000000..781fe11c10 --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/mod.rs @@ -0,0 +1 @@ +pub mod zelda; \ No newline at end of file diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/zelda.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/zelda.rs new file mode 100644 index 0000000000..76a8571cc6 --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/test_cases/zelda.rs @@ -0,0 +1,31 @@ +use crate::plans::model_weights::evaluate::asserts::assert_search_term_contains_titles; +use elasticsearch::Elasticsearch; +use errors::AnyhowResult; + +pub async fn test_zelda_searches( + client: &Elasticsearch +) -> AnyhowResult<()> { + + assert_search_term_contains_titles(client, "zeld", &vec![ + "Link (Zelda CDI) (Français)", + "Zelda (BOTW Español Latino)", + "Zelda (BOTW) (Ita)", + "Zelda (BOTW)", + "Zelda (Breath of the Wild)", + "Zelda Rubinstein", + "Zelda. (The Legend of Zelda: Breath of the Wild, Castillian Spanish.) Versión MEJORADA.", + ]).await?; + + assert_search_term_contains_titles(client, "zelad", &vec![ + "Link (Zelda CDI) (Français)", + "Zelda (BOTW Español Latino)", + "Zelda (BOTW) (Ita)", + "Zelda (BOTW)", + "Zelda (Breath of the Wild)", + "Zelda Rubinstein", + "Zelda. (The Legend of Zelda: Breath of the Wild, Castillian Spanish.) Versión MEJORADA.", + ]).await?; + + Ok(()) +} + diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/titles.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/titles.rs new file mode 100644 index 0000000000..bc818e57d2 --- /dev/null +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/evaluate/titles.rs @@ -0,0 +1,13 @@ +use std::collections::HashSet; +use elasticsearch_schema::documents::model_weight_document::ModelWeightDocument; + +pub fn print_titles(results: &Vec) { + for result in results { + println!(" - result: {:#?} ({:?})", result.title, result.token); + } +} + +pub fn to_title_set(results: &Vec) -> HashSet { + results.iter().map(|result| result.title.clone()).collect() +} + diff --git a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/mod.rs b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/mod.rs index 9473e0f3b9..1fbef94a50 100644 --- a/crates/cli/production/elasticsearch-cli/src/plans/model_weights/mod.rs +++ b/crates/cli/production/elasticsearch-cli/src/plans/model_weights/mod.rs @@ -1,2 +1,3 @@ -pub mod test_search_model_weights_documents; -pub mod create_all_model_weight_documents; \ No newline at end of file +pub mod create_all_model_weight_documents; +pub mod evaluate; +pub mod test_search_model_weights_documents; \ No newline at end of file diff --git a/crates/lib/elasticsearch_schema/src/searches/search_model_weights/min_score.rs b/crates/lib/elasticsearch_schema/src/searches/search_model_weights/min_score.rs index abc5fd9d8a..b2c69b3b16 100644 --- a/crates/lib/elasticsearch_schema/src/searches/search_model_weights/min_score.rs +++ b/crates/lib/elasticsearch_schema/src/searches/search_model_weights/min_score.rs @@ -21,12 +21,12 @@ pub fn add_min_score( } let minimum_score = match search_term.len() { - 1..=3 => 1, - 4 => 5, - 5 => 10, - 6 => 30, - 7 => 50, - _ => 100, + 1..=3 => 0, + 4 => 1, + 5 => 3, + 6 => 5, + 7 => 7, + _ => 10, }; if let Some(mut object) = query.as_object_mut() { diff --git a/crates/my-workspace-hack/Cargo.toml b/crates/my-workspace-hack/Cargo.toml index 996076aec5..6e1bff06af 100644 --- a/crates/my-workspace-hack/Cargo.toml +++ b/crates/my-workspace-hack/Cargo.toml @@ -4,6 +4,7 @@ [package] name = "my-workspace-hack" +edition = "2018" version = "0.1.0" description = "workspace-hack package, managed by hakari" # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.