refactor / cleanup for legibility

crates/api_clients/grok_client/src/requests/index_page/get_index_page_and_scripts.rs

@@ -1,9 +1,9 @@
 use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
 use crate::error::grok_client_error::GrokClientError;
 use crate::error::grok_error::GrokError;
+use crate::requests::index_page::index_parsers::parse_index_script_list::parse_index_svg_paths;
 use crate::requests::index_page::requests::get_index_page_scripts_with_client::{get_index_page_scripts_with_client, GetIndexPageScriptsArgs};
 use crate::requests::index_page::requests::get_index_page_with_client::{get_index_page_with_client, GetIndexPageWithClientArgs};
-use crate::requests::index_page::utils::parse_scripts_from_index_html::parse_scripts_from_index_html;
 use crate::utils::create_firefox_client::create_firefox_client;
 use log::info;
 use std::collections::HashMap;
@@ -38,7 +38,7 @@ pub async fn get_index_page_and_scripts(args: GetIndexPageAndScriptsArgs<'_>) ->
     cookie: args.cookie,
   }).await?;
 
-  let scripts = parse_scripts_from_index_html(&index.body);
+  let scripts = parse_index_svg_paths(&index.body);
 
   info!("Fetching {} scripts...", scripts.len());
 

crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs

@@ -1,9 +1,9 @@
 use crate::error::grok_client_error::GrokClientError;
 use crate::error::grok_error::GrokError;
+use crate::requests::index_page::index_parsers::parse_index_on_demand_script::parse_index_on_demand_script;
+use crate::requests::index_page::index_parsers::parse_index_svg_paths::parse_svg_paths_from_index_html;
 use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs};
 use crate::requests::index_page::utils::parse_numbers_from_xsid_script::parse_numbers_from_xsid_script;
-use crate::requests::index_page::utils::parse_on_demand_script_from_index_html::parse_on_demand_script_from_index_html;
-use crate::requests::index_page::utils::parse_svg_paths_from_index_html::parse_svg_paths_from_index_html;
 use crate::requests::index_page::utils::verification_token_to_loading_anim::LoadingAnim;
 use wreq::Client;
 
@@ -30,7 +30,7 @@ pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result<NumbersAndSvg, Grok
   let script_link = match args.xsid_script_id {
     "ondemand.s" => {
       // TODO: Not sure this is relevant anymore. Not a full or correct implementation of this branch.
-      let on_demand = parse_on_demand_script_from_index_html(&args.html);
+      let on_demand = parse_index_on_demand_script(&args.html);
       let on_demand = on_demand.unwrap_or_else(|| "".to_string());
       format!("https://abs.twimg.com/responsive-web/client-web/ondemand.s.{}a.js", on_demand)
     },
@@ -64,8 +64,8 @@ pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result<NumbersAndSvg, Grok
 mod tests {
   use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs};
   use crate::requests::index_page::get_numbers::{get_numbers, GetNumbersArgs};
+  use crate::requests::index_page::index_parsers::parse_index_verification_token::parse_verification_token_from_index_html;
   use crate::requests::index_page::utils::find_script_actions_and_xsid::find_script_actions_and_xsid;
-  use crate::requests::index_page::utils::parse_verification_token_from_index_html::parse_verification_token_from_index_html;
   use crate::requests::index_page::utils::verification_token_to_loading_anim::verification_token_to_loading_anim;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use errors::AnyhowResult;

crates/api_clients/grok_client/src/requests/index_page/index_parsers/mod.rs

@@ -0,0 +1,5 @@
+pub mod parse_index_baggage;
+pub mod parse_index_on_demand_script;
+pub mod parse_index_script_list;
+pub mod parse_index_svg_paths;
+pub mod parse_index_verification_token;

crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_baggage.rs

@@ -9,7 +9,7 @@ static BAGGAGE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
 });
 
 
-pub fn parse_baggage_from_index_html(html: &str) -> Option<String> {
+pub fn parse_index_baggage(html: &str) -> Option<String> {
   let document = Html::parse_document(html);
   let selected = document.select(&BAGGAGE_SELECTOR);
   let mut values = selected
@@ -22,7 +22,7 @@ pub fn parse_baggage_from_index_html(html: &str) -> Option<String> {
 #[cfg(test)]
 mod tests {
   use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::utils::parse_baggage_from_index_html::parse_baggage_from_index_html;
+  use crate::requests::index_page::index_parsers::parse_index_baggage::parse_index_baggage;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use errors::AnyhowResult;
 
@@ -34,7 +34,7 @@ mod tests {
       cookie: &cookie,
     }).await?;
 
-    let baggage = parse_baggage_from_index_html(&index.body);
+    let baggage = parse_index_baggage(&index.body);
 
     println!("Baggage: {:?}", baggage);
 

crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_on_demand_script.rs

@@ -6,7 +6,7 @@ static ON_DEMAND_REGEX : Lazy<Regex> = Lazy::new(|| {
       .expect("Regex should parse")
 });
 
-pub fn parse_on_demand_script_from_index_html(html: &str) -> Option<String> {
+pub fn parse_index_on_demand_script(html: &str) -> Option<String> {
   let on_demand = scrape_on_demand_via_regex(html);
 
   if let Some(on_demand) = on_demand {
@@ -26,7 +26,7 @@ fn scrape_on_demand_via_regex(html: &str) -> Option<String> {
 #[cfg(test)]
 mod tests {
   use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::utils::parse_on_demand_script_from_index_html::parse_on_demand_script_from_index_html;
+  use crate::requests::index_page::index_parsers::parse_index_on_demand_script::parse_index_on_demand_script;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use errors::AnyhowResult;
 
@@ -38,7 +38,7 @@ mod tests {
       cookie: &cookie,
     }).await?;
 
-    let result = parse_on_demand_script_from_index_html(&index.body);
+    let result = parse_index_on_demand_script(&index.body);
     println!("{:?}", result);
 
     assert_eq!(1, 2);

crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_script_list.rs

@@ -9,7 +9,7 @@ static SCRIPT_SELECTOR: Lazy<Selector> = Lazy::new(|| {
 });
 
 
-pub fn parse_scripts_from_index_html(html: &str) -> Vec<String> {
+pub fn parse_index_svg_paths(html: &str) -> Vec<String> {
   let document = Html::parse_document(html);
   let selected = document.select(&SCRIPT_SELECTOR);
   selected
@@ -22,9 +22,9 @@ pub fn parse_scripts_from_index_html(html: &str) -> Vec<String> {
 #[cfg(test)]
 mod tests {
   use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
+  use crate::requests::index_page::index_parsers::parse_index_script_list::parse_index_svg_paths;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use errors::AnyhowResult;
-  use crate::requests::index_page::utils::parse_scripts_from_index_html::parse_scripts_from_index_html;
 
   #[tokio::test]
   #[ignore] // Manual test invocation
@@ -34,7 +34,7 @@ mod tests {
       cookie: &cookie,
     }).await?;
 
-    let scripts = parse_scripts_from_index_html(&index.body);
+    let scripts = parse_index_svg_paths(&index.body);
 
     println!("Script count: {:?}", scripts.len());
 

crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_svg_paths.rs

@@ -25,7 +25,7 @@ pub fn parse_svg_paths_from_index_html(html: &str) -> Vec<String> {
 #[cfg(test)]
 mod tests {
   use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::utils::parse_svg_paths_from_index_html::parse_svg_paths_from_index_html;
+  use crate::requests::index_page::index_parsers::parse_index_svg_paths::parse_svg_paths_from_index_html;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use errors::AnyhowResult;
 

crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_verification_token.rs

@@ -61,7 +61,7 @@ fn scrape_meta_tag_via_parsing(html: &str) -> Option<String> {
 #[cfg(test)]
 mod tests {
   use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::utils::parse_verification_token_from_index_html::parse_verification_token_from_index_html;
+  use crate::requests::index_page::index_parsers::parse_index_verification_token::parse_verification_token_from_index_html;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use errors::AnyhowResult;
 

crates/api_clients/grok_client/src/requests/index_page/mod.rs

@@ -1,5 +1,6 @@
 pub (super) mod requests;
 pub mod get_index_page;
 pub mod get_index_page_and_scripts;
-pub mod utils;
 pub mod get_numbers;
+pub mod index_parsers;
+pub mod utils;

crates/api_clients/grok_client/src/requests/index_page/requests/get_index_page_scripts_with_client.rs

@@ -41,7 +41,7 @@ pub async fn get_index_page_scripts_with_client(args: GetIndexPageScriptsArgs<'_
 mod tests {
   use crate::requests::index_page::requests::get_index_page_scripts_with_client::{get_index_page_scripts_with_client, GetIndexPageScriptsArgs};
   use crate::requests::index_page::requests::get_index_page_with_client::{get_index_page_with_client, GetIndexPageWithClientArgs};
-  use crate::requests::index_page::utils::parse_scripts_from_index_html::parse_scripts_from_index_html;
+  use crate::requests::index_page::index_parsers::parse_index_script_list::parse_index_svg_paths;
   use crate::test_utils::get_test_cookies::get_test_cookies;
   use crate::utils::create_firefox_client::create_firefox_client;
   use errors::AnyhowResult;
@@ -59,7 +59,7 @@ mod tests {
       cookie: &cookie,
     }).await?;
 
-    let scripts = parse_scripts_from_index_html(&index.body);
+    let scripts = parse_index_svg_paths(&index.body);
 
     let scripts = get_index_page_scripts_with_client(GetIndexPageScriptsArgs {
       client: &client,

crates/api_clients/grok_client/src/requests/index_page/utils/mod.rs

@@ -1,8 +1,3 @@
 pub mod find_script_actions_and_xsid;
 pub mod parse_numbers_from_xsid_script;
-pub mod parse_on_demand_script_from_index_html;
-pub mod parse_scripts_from_index_html;
-pub mod parse_svg_paths_from_index_html;
-pub mod parse_verification_token_from_index_html;
 pub mod verification_token_to_loading_anim;
-mod parse_baggage_from_index_html;