From 4d89ce130bf958a3420532e4fe3937a478f7c13f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 29 Oct 2025 00:48:20 -0400 Subject: [PATCH] wip --- .../src/recipes/request_client_secrets.rs | 16 +++++---- .../src/requests/index_page/get_numbers.rs | 35 +++---------------- .../requests/index_page/get_xsid_script.rs | 6 ++-- .../index_parsers/parse_index_sentry_trace.rs | 8 +++-- .../src/requests/index_page/pieces/mod.rs | 4 ++- .../index_page/pieces/sentry_trace.rs | 4 +++ .../index_page/pieces/xsid_numbers.rs | 6 ++++ .../utils/parse_numbers_from_xsid_script.rs | 11 ++++-- 8 files changed, 44 insertions(+), 46 deletions(-) create mode 100644 crates/api_clients/grok_client/src/requests/index_page/pieces/sentry_trace.rs create mode 100644 crates/api_clients/grok_client/src/requests/index_page/pieces/xsid_numbers.rs diff --git a/crates/api_clients/grok_client/src/recipes/request_client_secrets.rs b/crates/api_clients/grok_client/src/recipes/request_client_secrets.rs index 9926b2b60f..3f919f0daa 100644 --- a/crates/api_clients/grok_client/src/recipes/request_client_secrets.rs +++ b/crates/api_clients/grok_client/src/recipes/request_client_secrets.rs @@ -6,6 +6,7 @@ use crate::requests::index_page::index_parsers::parse_index_sentry_trace::parse_ use crate::requests::index_page::index_parsers::parse_index_svg_paths::parse_svg_paths_from_index_html; use crate::requests::index_page::index_parsers::parse_index_verification_token::parse_index_verification_token; use crate::requests::index_page::pieces::baggage::Baggage; +use crate::requests::index_page::pieces::sentry_trace::SentryTrace; use crate::requests::index_page::pieces::svg_path_data::SvgPathData; use crate::requests::index_page::pieces::verification_token::VerificationToken; use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::convert_verification_token_to_loading_anim; @@ -26,12 +27,14 @@ pub struct ClientSecrets { pub baggage: Baggage, /// From the index HTML meta tag - pub sentry_trace: String, + pub sentry_trace: SentryTrace, /// From the index HTML meta tag pub verification_token: VerificationToken, - + /// The SVG path data to use (chosen via verification_token) + /// We selected one of four-ish possible SVG paths (of length >= 200) by + /// the `verification token -> animation index` algo. pub svg_path: SvgPathData, /// Scripts (for later use) @@ -60,18 +63,19 @@ pub async fn request_client_secrets(args: RequestClientSecretsArgs<'_>) -> Resul message: "Index did not include verification token.".to_string() })?; + // NB: We'll only use one of the several SVG paths let svg_paths = parse_svg_paths_from_index_html(&payloads.index_body_html); - + if svg_paths.is_empty() { return Err(GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData { message: "Index did not include any SVG paths.".to_string(), }.into()); } - + let loading_anim = convert_verification_token_to_loading_anim(&verification_token)?; - + let svg_path = select_svg_path_by_loading_anim(&svg_paths, &loading_anim)?; - + let actions_and_xsid_script = find_script_actions_and_xsid_script_path(&payloads.scripts)?; // self.svg_data, self.numbers = Parser.parse_values(c_request.text, self.anim, self.xsid_script) diff --git a/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs b/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs index 5d0c9fca16..0d1366cddc 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/get_numbers.rs @@ -1,13 +1,10 @@ -use crate::error::grok_client_error::GrokClientError; use crate::error::grok_error::GrokError; use crate::requests::index_page::index_parsers::parse_index_on_demand_script::parse_index_on_demand_script; -use crate::requests::index_page::index_parsers::parse_index_svg_paths::parse_svg_paths_from_index_html; -use crate::requests::index_page::pieces::svg_path_data::SvgPathData; +use crate::requests::index_page::pieces::xsid_numbers::XsidNumbers; use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs}; -use crate::requests::index_page::utils::parse_numbers_from_xsid_script::parse_numbers_from_xsid_script; use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::LoadingAnim; +use crate::requests::index_page::utils::parse_numbers_from_xsid_script::parse_numbers_from_xsid_script; use wreq::Client; - // self.svg_data, self.numbers = Parser.parse_values(c_request.text, self.anim, self.xsid_script) pub struct GetNumbersArgs<'a> { @@ -18,15 +15,7 @@ pub struct GetNumbersArgs<'a> { pub xsid_script_id: &'a str, } -#[derive(Clone, Debug)] -pub struct NumbersAndSvg { - pub numbers: Vec, - - /// We selected one of four possible SVG paths (of length >= 200) by the animation index. - pub svg_data: SvgPathData, -} - -pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result { +pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result { let script_link = match args.xsid_script_id { "ondemand.s" => { @@ -44,21 +33,7 @@ pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result s.clone(), - None => return Err(GrokClientError::ScriptLogicOutOfDate.into()), - }; - - Ok(NumbersAndSvg { - numbers, - svg_data, - }) + Ok(parse_numbers_from_xsid_script(&xsid_script_body)) } #[cfg(test)] @@ -66,8 +41,8 @@ mod tests { use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs}; use crate::requests::index_page::get_numbers::{get_numbers, GetNumbersArgs}; use crate::requests::index_page::index_parsers::parse_index_verification_token::parse_index_verification_token; - use crate::requests::index_page::utils::find_script_actions_and_xsid_script_path::find_script_actions_and_xsid_script_path; use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::convert_verification_token_to_loading_anim; + use crate::requests::index_page::utils::find_script_actions_and_xsid_script_path::find_script_actions_and_xsid_script_path; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; diff --git a/crates/api_clients/grok_client/src/requests/index_page/get_xsid_script.rs b/crates/api_clients/grok_client/src/requests/index_page/get_xsid_script.rs index 68d54eb90e..5b02bb41e6 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/get_xsid_script.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/get_xsid_script.rs @@ -1,9 +1,9 @@ use crate::error::grok_error::GrokError; use crate::requests::index_page::index_parsers::parse_index_on_demand_script::parse_index_on_demand_script; +use crate::requests::index_page::pieces::xsid_numbers::XsidNumbers; use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs}; use crate::requests::index_page::utils::parse_numbers_from_xsid_script::parse_numbers_from_xsid_script; use wreq::Client; - // self.svg_data, self.numbers = Parser.parse_values(c_request.text, self.anim, self.xsid_script) pub struct GetXsidScriptArgs<'a> { @@ -15,7 +15,7 @@ pub struct GetXsidScriptArgs<'a> { #[derive(Clone, Debug)] pub struct Numbers { - pub numbers: Vec, + pub numbers: XsidNumbers, pub xsid_script_body: String, } @@ -50,8 +50,8 @@ mod tests { use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs}; use crate::requests::index_page::get_numbers::{get_numbers, GetNumbersArgs}; use crate::requests::index_page::index_parsers::parse_index_verification_token::parse_index_verification_token; - use crate::requests::index_page::utils::find_script_actions_and_xsid_script_path::find_script_actions_and_xsid_script_path; use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::convert_verification_token_to_loading_anim; + use crate::requests::index_page::utils::find_script_actions_and_xsid_script_path::find_script_actions_and_xsid_script_path; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; diff --git a/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_sentry_trace.rs b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_sentry_trace.rs index 2e604975da..35ddc533f6 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_sentry_trace.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/index_parsers/parse_index_sentry_trace.rs @@ -1,6 +1,6 @@ +use crate::requests::index_page::pieces::sentry_trace::SentryTrace; use once_cell::sync::Lazy; use scraper::{Html, Selector}; - // static SENTRY_TRACE_SELECTOR : Lazy = Lazy::new(|| { @@ -11,8 +11,10 @@ static SENTRY_TRACE_SELECTOR : Lazy = Lazy::new(|| { /// Extract the sentry trace, then process it. /// The Python library discards the latter half of the trace. -pub fn parse_index_sentry_trace(html: &str) -> Option { - get_meta_value(html).map(|trace| split_sentry_trace(&trace)) +pub fn parse_index_sentry_trace(html: &str) -> Option { + get_meta_value(html) + .map(|trace| split_sentry_trace(&trace)) + .map(|trace| SentryTrace(trace)) } fn get_meta_value(html: &str) -> Option { diff --git a/crates/api_clients/grok_client/src/requests/index_page/pieces/mod.rs b/crates/api_clients/grok_client/src/requests/index_page/pieces/mod.rs index 980551a465..a1c2fcd83f 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/pieces/mod.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/pieces/mod.rs @@ -1,3 +1,5 @@ pub mod baggage; +pub mod sentry_trace; pub mod svg_path_data; -pub mod verification_token; \ No newline at end of file +pub mod verification_token; +pub mod xsid_numbers; \ No newline at end of file diff --git a/crates/api_clients/grok_client/src/requests/index_page/pieces/sentry_trace.rs b/crates/api_clients/grok_client/src/requests/index_page/pieces/sentry_trace.rs new file mode 100644 index 0000000000..3a8ad9f330 --- /dev/null +++ b/crates/api_clients/grok_client/src/requests/index_page/pieces/sentry_trace.rs @@ -0,0 +1,4 @@ + +/// "sentry trace" data from the tags +#[derive(Debug, Clone)] +pub struct SentryTrace(pub (crate) String); \ No newline at end of file diff --git a/crates/api_clients/grok_client/src/requests/index_page/pieces/xsid_numbers.rs b/crates/api_clients/grok_client/src/requests/index_page/pieces/xsid_numbers.rs new file mode 100644 index 0000000000..484b010066 --- /dev/null +++ b/crates/api_clients/grok_client/src/requests/index_page/pieces/xsid_numbers.rs @@ -0,0 +1,6 @@ + +/// Magic numbers pulled from Javascript +#[derive(Clone, Debug)] +pub struct XsidNumbers { + pub numbers: Vec, +} diff --git a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_numbers_from_xsid_script.rs b/crates/api_clients/grok_client/src/requests/index_page/utils/parse_numbers_from_xsid_script.rs index 3713d8a9ae..18ce229f96 100644 --- a/crates/api_clients/grok_client/src/requests/index_page/utils/parse_numbers_from_xsid_script.rs +++ b/crates/api_clients/grok_client/src/requests/index_page/utils/parse_numbers_from_xsid_script.rs @@ -1,3 +1,4 @@ +use crate::requests::index_page::pieces::xsid_numbers::XsidNumbers; use log::error; use once_cell::sync::Lazy; use regex::Regex; @@ -8,8 +9,8 @@ static NUMBERS_REGEX : Lazy = Lazy::new(|| { .expect("Regex should parse") }); -pub fn parse_numbers_from_xsid_script(script_body: &str) -> Vec { - NUMBERS_REGEX.captures_iter(&script_body) +pub fn parse_numbers_from_xsid_script(script_body: &str) -> XsidNumbers { + let numbers = NUMBERS_REGEX.captures_iter(&script_body) .flat_map(|captures| captures.get(1).map(|m| m.as_str().to_string())) .flat_map(|num| match u32::from_str(&num) { Ok(num) => Some(num), @@ -18,5 +19,9 @@ pub fn parse_numbers_from_xsid_script(script_body: &str) -> Vec { None } }) - .collect::<_>() + .collect::<_>(); + + XsidNumbers { + numbers, + } }