From 722c031a09fa038716fa78e96da993562f4e3b60 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 28 Oct 2025 04:23:52 -0400 Subject: [PATCH] refactor / cleanup for legibility --- .../src/requests/index_page/get_index_page_and_scripts.rs | 4 ++-- .../grok_client/src/requests/index_page/get_numbers.rs | 8 ++++---- .../src/requests/index_page/index_parsers/mod.rs | 5 +++++ .../parse_index_baggage.rs} | 6 +++--- .../parse_index_on_demand_script.rs} | 6 +++--- .../parse_index_script_list.rs} | 6 +++--- .../parse_index_svg_paths.rs} | 2 +- .../parse_index_verification_token.rs} | 2 +- .../grok_client/src/requests/index_page/mod.rs | 3 ++- .../requests/get_index_page_scripts_with_client.rs | 4 ++-- .../grok_client/src/requests/index_page/utils/mod.rs | 5 ----- 11 files changed, 26 insertions(+), 25 deletions(-) create mode 100644 crates/api_clients/grok_client/src/requests/index_page/index_parsers/mod.rs rename crates/api_clients/grok_client/src/requests/index_page/{utils/parse_baggage_from_index_html.rs => index_parsers/parse_index_baggage.rs} (83%) rename crates/api_clients/grok_client/src/requests/index_page/{utils/parse_on_demand_script_from_index_html.rs => index_parsers/parse_index_on_demand_script.rs} (78%) rename crates/api_clients/grok_client/src/requests/index_page/{utils/parse_scripts_from_index_html.rs => index_parsers/parse_index_script_list.rs} (83%) rename crates/api_clients/grok_client/src/requests/index_page/{utils/parse_svg_paths_from_index_html.rs => index_parsers/parse_index_svg_paths.rs} (93%) rename crates/api_clients/grok_client/src/requests/index_page/{utils/parse_verification_token_from_index_html.rs => index_parsers/parse_index_verification_token.rs} (95%) diff --git a/crates/api_clients/grok_client/src/requests/index_page/get_index_page_and_scripts.rs b/crates/api_clients/grok_client/src/requests/index_page/get_index_page_and_scripts.rs index b4d7b470f7..6d5a3a4c60 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/get_index_page_and_scripts.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/get_index_page_and_scripts.rs @@ -1,9 +1,9 @@ use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT; use crate::error::grok_client_error::GrokClientError; use crate::error::grok_error::GrokError; +use crate::requests::index_page::index_parsers::parse_index_script_list::parse_index_svg_paths; use crate::requests::index_page::requests::get_index_page_scripts_with_client::{get_index_page_scripts_with_client, GetIndexPageScriptsArgs}; use crate::requests::index_page::requests::get_index_page_with_client::{get_index_page_with_client, GetIndexPageWithClientArgs}; -use crate::requests::index_page::utils::parse_scripts_from_index_html::parse_scripts_from_index_html; use crate::utils::create_firefox_client::create_firefox_client; use log::info; use std::collections::HashMap; @@ -38,7 +38,7 @@ pub async fn get_index_page_and_scripts(args: GetIndexPageAndScriptsArgs<'_>) -> cookie: args.cookie, }).await?; - let scripts = parse_scripts_from_index_html(&index.body); + let scripts = parse_index_svg_paths(&index.body); info!("Fetching {} scripts...", scripts.len()); diff --git a/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs b/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs index 5566e430f2..be98e835b7 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs @@ -1,9 +1,9 @@ use crate::error::grok_client_error::GrokClientError; use crate::error::grok_error::GrokError; +use crate::requests::index_page::index_parsers::parse_index_on_demand_script::parse_index_on_demand_script; +use crate::requests::index_page::index_parsers::parse_index_svg_paths::parse_svg_paths_from_index_html; use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs}; use crate::requests::index_page::utils::parse_numbers_from_xsid_script::parse_numbers_from_xsid_script; -use crate::requests::index_page::utils::parse_on_demand_script_from_index_html::parse_on_demand_script_from_index_html; -use crate::requests::index_page::utils::parse_svg_paths_from_index_html::parse_svg_paths_from_index_html; use crate::requests::index_page::utils::verification_token_to_loading_anim::LoadingAnim; use wreq::Client; @@ -30,7 +30,7 @@ pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result { // TODO: Not sure this is relevant anymore. Not a full or correct implementation of this branch. - let on_demand = parse_on_demand_script_from_index_html(&args.html); + let on_demand = parse_index_on_demand_script(&args.html); let on_demand = on_demand.unwrap_or_else(|| "".to_string()); format!("https://abs.twimg.com/responsive-web/client-web/ondemand.s.{}a.js", on_demand) }, @@ -64,8 +64,8 @@ pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result = Lazy::new(|| { }); -pub fn parse_baggage_from_index_html(html: &str) -> Option { +pub fn parse_index_baggage(html: &str) -> Option { let document = Html::parse_document(html); let selected = document.select(&BAGGAGE_SELECTOR); let mut values = selected @@ -22,7 +22,7 @@ pub fn parse_baggage_from_index_html(html: &str) -> Option { #[cfg(test)] mod tests { use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs}; - use crate::requests::index_page::utils::parse_baggage_from_index_html::parse_baggage_from_index_html; + use crate::requests::index_page::index_parsers::parse_index_baggage::parse_index_baggage; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; @@ -34,7 +34,7 @@ mod tests { cookie: &cookie, }).await?; - let baggage = parse_baggage_from_index_html(&index.body); + let baggage = parse_index_baggage(&index.body); println!("Baggage: {:?}", baggage); diff --git a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_on_demand_script_from_index_html.rs b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_on_demand_script.rs similarity index 78% rename from crates/api_clients/grok_client/src/requests/index_page/utils/parse_on_demand_script_from_index_html.rs rename to crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_on_demand_script.rs index c0f32f062a..8d33bb2c81 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_on_demand_script_from_index_html.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_on_demand_script.rs @@ -6,7 +6,7 @@ static ON_DEMAND_REGEX : Lazy = Lazy::new(|| { .expect("Regex should parse") }); -pub fn parse_on_demand_script_from_index_html(html: &str) -> Option { +pub fn parse_index_on_demand_script(html: &str) -> Option { let on_demand = scrape_on_demand_via_regex(html); if let Some(on_demand) = on_demand { @@ -26,7 +26,7 @@ fn scrape_on_demand_via_regex(html: &str) -> Option { #[cfg(test)] mod tests { use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs}; - use crate::requests::index_page::utils::parse_on_demand_script_from_index_html::parse_on_demand_script_from_index_html; + use crate::requests::index_page::index_parsers::parse_index_on_demand_script::parse_index_on_demand_script; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; @@ -38,7 +38,7 @@ mod tests { cookie: &cookie, }).await?; - let result = parse_on_demand_script_from_index_html(&index.body); + let result = parse_index_on_demand_script(&index.body); println!("{:?}", result); assert_eq!(1, 2); diff --git a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_scripts_from_index_html.rs b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_script_list.rs similarity index 83% rename from crates/api_clients/grok_client/src/requests/index_page/utils/parse_scripts_from_index_html.rs rename to crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_script_list.rs index 3de986e9e0..3c7e749511 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_scripts_from_index_html.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_script_list.rs @@ -9,7 +9,7 @@ static SCRIPT_SELECTOR: Lazy = Lazy::new(|| { }); -pub fn parse_scripts_from_index_html(html: &str) -> Vec { +pub fn parse_index_svg_paths(html: &str) -> Vec { let document = Html::parse_document(html); let selected = document.select(&SCRIPT_SELECTOR); selected @@ -22,9 +22,9 @@ pub fn parse_scripts_from_index_html(html: &str) -> Vec { #[cfg(test)] mod tests { use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs}; + use crate::requests::index_page::index_parsers::parse_index_script_list::parse_index_svg_paths; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; - use crate::requests::index_page::utils::parse_scripts_from_index_html::parse_scripts_from_index_html; #[tokio::test] #[ignore] // Manual test invocation @@ -34,7 +34,7 @@ mod tests { cookie: &cookie, }).await?; - let scripts = parse_scripts_from_index_html(&index.body); + let scripts = parse_index_svg_paths(&index.body); println!("Script count: {:?}", scripts.len()); diff --git a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_svg_paths_from_index_html.rs b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_svg_paths.rs similarity index 93% rename from crates/api_clients/grok_client/src/requests/index_page/utils/parse_svg_paths_from_index_html.rs rename to crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_svg_paths.rs index 4a45d2c012..3e407a7943 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_svg_paths_from_index_html.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_svg_paths.rs @@ -25,7 +25,7 @@ pub fn parse_svg_paths_from_index_html(html: &str) -> Vec { #[cfg(test)] mod tests { use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs}; - use crate::requests::index_page::utils::parse_svg_paths_from_index_html::parse_svg_paths_from_index_html; + use crate::requests::index_page::index_parsers::parse_index_svg_paths::parse_svg_paths_from_index_html; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; diff --git a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_verification_token_from_index_html.rs b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_verification_token.rs similarity index 95% rename from crates/api_clients/grok_client/src/requests/index_page/utils/parse_verification_token_from_index_html.rs rename to crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_verification_token.rs index d7e8c79166..ae9b54950e 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_verification_token_from_index_html.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_verification_token.rs @@ -61,7 +61,7 @@ fn scrape_meta_tag_via_parsing(html: &str) -> Option { #[cfg(test)] mod tests { use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs}; - use crate::requests::index_page::utils::parse_verification_token_from_index_html::parse_verification_token_from_index_html; + use crate::requests::index_page::index_parsers::parse_index_verification_token::parse_verification_token_from_index_html; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; diff --git a/crates/api_clients/grok_client/src/requests/index_page/mod.rs b/crates/api_clients/grok_client/src/requests/index_page/mod.rs index 014e8221ef..50ec10588b 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/mod.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/mod.rs @@ -1,5 +1,6 @@ pub (super) mod requests; pub mod get_index_page; pub mod get_index_page_and_scripts; -pub mod utils; pub mod get_numbers; +pub mod index_parsers; +pub mod utils; diff --git a/crates/api_clients/grok_client/src/requests/index_page/requests/get_index_page_scripts_with_client.rs b/crates/api_clients/grok_client/src/requests/index_page/requests/get_index_page_scripts_with_client.rs index 92fa1f2e90..bd564f677e 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/requests/get_index_page_scripts_with_client.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/requests/get_index_page_scripts_with_client.rs @@ -41,7 +41,7 @@ pub async fn get_index_page_scripts_with_client(args: GetIndexPageScriptsArgs<'_ mod tests { use crate::requests::index_page::requests::get_index_page_scripts_with_client::{get_index_page_scripts_with_client, GetIndexPageScriptsArgs}; use crate::requests::index_page::requests::get_index_page_with_client::{get_index_page_with_client, GetIndexPageWithClientArgs}; - use crate::requests::index_page::utils::parse_scripts_from_index_html::parse_scripts_from_index_html; + use crate::requests::index_page::index_parsers::parse_index_script_list::parse_index_svg_paths; use crate::test_utils::get_test_cookies::get_test_cookies; use crate::utils::create_firefox_client::create_firefox_client; use errors::AnyhowResult; @@ -59,7 +59,7 @@ mod tests { cookie: &cookie, }).await?; - let scripts = parse_scripts_from_index_html(&index.body); + let scripts = parse_index_svg_paths(&index.body); let scripts = get_index_page_scripts_with_client(GetIndexPageScriptsArgs { client: &client, diff --git a/crates/api_clients/grok_client/src/requests/index_page/utils/mod.rs b/crates/api_clients/grok_client/src/requests/index_page/utils/mod.rs index 73eddfe2a9..ff7f448a17 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/utils/mod.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/utils/mod.rs @@ -1,8 +1,3 @@ pub mod find_script_actions_and_xsid; pub mod parse_numbers_from_xsid_script; -pub mod parse_on_demand_script_from_index_html; -pub mod parse_scripts_from_index_html; -pub mod parse_svg_paths_from_index_html; -pub mod parse_verification_token_from_index_html; pub mod verification_token_to_loading_anim; -mod parse_baggage_from_index_html; \ No newline at end of file