From 290edce1a350df218d64ee92a990f0d9bb0b1c1b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 20 Oct 2025 17:02:06 -0400 Subject: [PATCH] cleanup, new lib --- Cargo.lock | 10 +++ Cargo.toml | 2 + crates/api_clients/grok_client/Cargo.toml | 1 + .../src/error/grok_generic_api_error.rs | 4 ++ .../create_listen_websocket_new.rs | 59 +++++----------- ...eate_listen_websocket_raw_non_websocket.rs | 67 ------------------- .../mitm_proxy/chrome_listen_detail.txt | 0 .../mitm_proxy/chrome_listen_request.txt | 0 .../mitm_proxy/chrome_listen_response.txt | 0 .../mitm_proxy/firefox_listen_detail.txt | 0 .../mitm_proxy/firefox_listen_request.txt | 0 .../mitm_proxy/firefox_listen_response.txt | 0 .../{ => metadata}/mitm_proxy/notes.txt | 0 .../mitm_proxy/wreq_listen_detail.txt | 0 .../mitm_proxy/wreq_listen_request.txt | 0 .../mitm_proxy/wreq_listen_request_2.txt | 0 .../mitm_proxy/wreq_listen_request_3.txt | 0 .../mitm_proxy/wreq_listen_request_raw_1.txt | 0 .../mitm_proxy/wreq_listen_response.txt | 0 .../tls_fingerprint_checking.rs | 0 .../src/requests/listen_websocket/mod.rs | 8 +-- .../listen_websocket/tls_check/mod.rs | 1 - crates/lib/cloudflare_mitigation/Cargo.toml | 26 +++++++ .../firefox_websocket_http_1_1_headers.rs | 33 +++++++++ .../cloudflare_mitigation/src/headers/mod.rs | 1 + crates/lib/cloudflare_mitigation/src/lib.rs | 1 + 26 files changed, 97 insertions(+), 116 deletions(-) delete mode 100644 crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_raw_non_websocket.rs rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/chrome_listen_detail.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/chrome_listen_request.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/chrome_listen_response.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/firefox_listen_detail.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/firefox_listen_request.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/firefox_listen_response.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/notes.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/wreq_listen_detail.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/wreq_listen_request.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/wreq_listen_request_2.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/wreq_listen_request_3.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/wreq_listen_request_raw_1.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{ => metadata}/mitm_proxy/wreq_listen_response.txt (100%) rename crates/api_clients/grok_client/src/requests/listen_websocket/{tls_check => metadata}/tls_fingerprint_checking.rs (100%) delete mode 100644 crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/mod.rs create mode 100644 crates/lib/cloudflare_mitigation/Cargo.toml create mode 100644 crates/lib/cloudflare_mitigation/src/headers/firefox_websocket_http_1_1_headers.rs create mode 100644 crates/lib/cloudflare_mitigation/src/headers/mod.rs create mode 100644 crates/lib/cloudflare_mitigation/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ec0a31f498..ba2126b324 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2180,6 +2180,15 @@ dependencies = [ "workspace-build-acceleration", ] +[[package]] +name = "cloudflare_mitigation" +version = "0.0.1" +dependencies = [ + "once_cell", + "workspace-build-acceleration", + "wreq", +] + [[package]] name = "cmake" version = "0.1.54" @@ -4666,6 +4675,7 @@ dependencies = [ "anyhow", "chrono", "cloudflare_errors", + "cloudflare_mitigation", "cookie_store 0.0.1", "env", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index ac8097ffa1..59f27455df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "crates/lib/caching/memory_store", "crates/lib/caching/redis_caching", "crates/lib/cloudflare_errors", + "crates/lib/cloudflare_mitigation", "crates/lib/concurrency", "crates/lib/cookie_store", "crates/lib/cookies", @@ -187,6 +188,7 @@ artcraft_api_defs = { path = "crates/api_clients/artcraft/artcraft_api_defs" } build_metadata = { path = "crates/lib/build_metadata" } cloud_storage = { path = "crates/api_clients/cloud_storage" } cloudflare_errors = { path = "crates/lib/cloudflare_errors" } +cloudflare_mitigation = { path = "crates/lib/cloudflare_mitigation" } component_traits = { path = "crates/service/plugins/component_traits" } cookie_store = { path = "crates/lib/cookie_store" } crockford = { path = "crates/lib/data/crockford" } diff --git a/crates/api_clients/grok_client/Cargo.toml b/crates/api_clients/grok_client/Cargo.toml index e220ed53fa..5a4eb481f9 100644 --- a/crates/api_clients/grok_client/Cargo.toml +++ b/crates/api_clients/grok_client/Cargo.toml @@ -17,6 +17,7 @@ workspace-build-acceleration.workspace = true # Internal cloudflare_errors.workspace = true +cloudflare_mitigation.workspace = true cookie_store.workspace = true errors.workspace = true diff --git a/crates/api_clients/grok_client/src/error/grok_generic_api_error.rs b/crates/api_clients/grok_client/src/error/grok_generic_api_error.rs index 2d90922d37..736cf090bd 100644 --- a/crates/api_clients/grok_client/src/error/grok_generic_api_error.rs +++ b/crates/api_clients/grok_client/src/error/grok_generic_api_error.rs @@ -23,6 +23,9 @@ pub enum GrokGenericApiError { body: String, }, + /// An error upgrading the connection to a websocket. + WreqWebsocketUpgradeError(wreq::Error), + /// An uncaught error from the API client. WreqError(wreq::Error), } @@ -36,6 +39,7 @@ impl Display for GrokGenericApiError { Self::SerdeResponseParseErrorWithBody(err, body) => write!(f, "Failed to parse response body: {:?}. Body: {}", err, body), Self::SerdeParseErrorWithBodyOnNon200(err, body) => write!(f, "Failed to parse non-200 response body: {:?}. Body: {}", err, body), Self::UncategorizedBadResponseWithStatusAndBody { status_code, body } => write!(f, "Uncategorized bad response: status code {}, body: {}", status_code, body), + Self::WreqWebsocketUpgradeError(err) => write!(f, "Websocket upgrade error: {}", err), Self::WreqError(err) => write!(f, "Wreq client error: {}", err), } } diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_new.rs b/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_new.rs index 4fce740bcc..2ad561db3d 100644 --- a/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_new.rs +++ b/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_new.rs @@ -1,55 +1,32 @@ +use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT; use crate::error::grok_client_error::GrokClientError; use crate::error::grok_error::GrokError; +use crate::error::grok_generic_api_error::GrokGenericApiError; use crate::requests::listen_websocket::cookies::SESSION_COOKIES_WITHOUT_CF_CLEARANCE; +use cloudflare_mitigation::headers::firefox_websocket_http_1_1_headers::get_firefox_websocket_http_1_1_headers; use std::ops::Deref; use std::time::Duration; +use log::info; use tokio::io::AsyncReadExt; -use wreq::{Client, Proxy}; use wreq::header::{ACCEPT, ACCEPT_LANGUAGE, CACHE_CONTROL, COOKIE, ORIGIN, PRAGMA, SEC_WEBSOCKET_EXTENSIONS, USER_AGENT}; +use wreq::{Client, Proxy}; use wreq_util::Emulation; -use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT; const WEBSOCKET_URL: &str = "wss://grok.com/ws/imagine/listen"; pub async fn create_listen_websocket_new() -> Result<(), GrokError> { - /* - Wreq Firefox vs Real Firefox Private Mode: - - - wreq has two extensions real firefox doesn't: - - "session_ticket (35) (IANA)" - - "psk_key_exchange_modes (45) (IANA)" - - wreq has "psk_key_exchange_mode": "AQ==", whereas real firefox private mode is null - - wreq has "session_ticket_supported": true, whereas real firefox private mode is false - - Wreq Firefox is IDENTICAL to Real Firefox Normal Mode (non-private) mode. - So why is it not working!? - */ - let mut client_builder = Client::builder() .emulation(Emulation::Firefox143) .connection_verbose(true) .connect_timeout(Duration::from_secs(10)); - let mut proxy = None; - - proxy = Some(Proxy::https("http://127.0.0.1:8080") - .map_err(|err| GrokClientError::WreqClientError(err))?); - - if let Some(proxy) = proxy { - client_builder = client_builder - .cert_verification(false) - .proxy(proxy); - } - let client = client_builder .build() .map_err(|err| GrokClientError::WreqClientError(err))?; - //.http1_only() // NB: Not needed - websockets are sent over HTTP/1.1 without this configuration - //.cookie_store(true) - let builder = client.websocket(WEBSOCKET_URL) - //.header("Host", "grok.com") .default_headers(false) + .orig_headers(get_firefox_websocket_http_1_1_headers()) .header(COOKIE, SESSION_COOKIES_WITHOUT_CF_CLEARANCE) .header(ORIGIN,"https://grok.com") .header(USER_AGENT, FIREFOX_143_MAC_USER_AGENT) @@ -60,29 +37,25 @@ pub async fn create_listen_websocket_new() -> Result<(), GrokError> { .header("Sec-Fetch-Mode", "websocket") .header("Sec-Fetch-Site", "same-origin") .header(PRAGMA, "no-cache") - .header(CACHE_CONTROL, "no-cache") - ; - - // TODO: Connect: keep-alive --> keep-alive, Upgrade - // TODO: Accept-Encoding: "gzip, deflate, br, zstd" - // TODO: Camel case header names - // TODO: Header orders! + .header(CACHE_CONTROL, "no-cache"); let response = builder.send() .await .map_err(|err| GrokClientError::WreqClientError(err))?; - println!("Response version: {:?}", response.version()); - let status = response.status(); - println!("Status: {}", status); - for cookie in response.cookies() { - println!("Cookie: {}={}", cookie.name(), cookie.value()); + info!("WebSocket Response Status: {}", status); + + if !status.is_success() || status.as_u16() != 101 { } - for (name, value) in response.headers() { - println!("Header: {}: {}", name.as_str(), value.to_str().unwrap()); + let mut websocket = response.into_websocket() + .await + .map_err(|err| GrokGenericApiError::WreqWebsocketUpgradeError(err))?; + + if let Some(protocol) = websocket.protocol() { + println!("WebSocket subprotocol: {:?}", protocol); } Ok(()) diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_raw_non_websocket.rs b/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_raw_non_websocket.rs deleted file mode 100644 index 062103aac3..0000000000 --- a/crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_raw_non_websocket.rs +++ /dev/null @@ -1,67 +0,0 @@ -use crate::error::grok_client_error::GrokClientError; -use crate::error::grok_error::GrokError; -use std::ops::Deref; -use std::time::Duration; -use tokio::io::AsyncReadExt; -use wreq::Client; -use wreq_util::Emulation; - -const WEBSOCKET_WSS_URL: &str = "wss://grok.com/ws/imagine/listen"; -const WEBSOCKET_HTTPS_URL: &str = "https://grok.com/ws/imagine/listen"; - -pub async fn create_listen_websocket_raw_non_websocket() -> Result<(), GrokError> { - let client = Client::builder() - .emulation(Emulation::Chrome140) - .http1_only() - .connection_verbose(true) - .connect_timeout(Duration::from_secs(10)) - .build() - .map_err(|err| GrokClientError::WreqClientError(err))?; - - //.http1_only() // NB: Not needed - websockets are sent over HTTP/1.1 without this configuration - //.cookie_store(true) - - let cookies = "_ga=GA1.1.1232202746.1760710013; i18nextLng=en; sso=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzZXNzaW9uX2lkIjoiOGU3MDFiNzctOTdkNC00ZjM0LWExOTctOWFmMDU1MzY3NDAwIn0.-a6x0InxbGzfTVfUlrdzxskxCnvMDI8lC90z4wHeGIk; sso-rw=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzZXNzaW9uX2lkIjoiOGU3MDFiNzctOTdkNC00ZjM0LWExOTctOWFmMDU1MzY3NDAwIn0.-a6x0InxbGzfTVfUlrdzxskxCnvMDI8lC90z4wHeGIk; stblid=b3331fc1-45d7-466b-83df-67427c0b2367; mp_ea93da913ddb66b6372b89d97b1029ac_mixpanel=%7B%22distinct_id%22%3A%2285980643-ffab-4984-a3de-59a608c47d7f%22%2C%22%24device_id%22%3A%2279ce237a-a0f3-4913-bf4b-519ac8a98263%22%2C%22%24initial_referrer%22%3A%22%24direct%22%2C%22%24initial_referring_domain%22%3A%22%24direct%22%2C%22__mps%22%3A%7B%7D%2C%22__mpso%22%3A%7B%7D%2C%22__mpus%22%3A%7B%7D%2C%22__mpa%22%3A%7B%7D%2C%22__mpu%22%3A%7B%7D%2C%22__mpr%22%3A%5B%5D%2C%22__mpap%22%3A%5B%5D%2C%22%24user_id%22%3A%2285980643-ffab-4984-a3de-59a608c47d7f%22%7D; cf_clearance=e7.LXFbmc.U81rZIqxuBfBe88yuBZMXEP.zwxNkbOxw-1760724394-1.2.1.1-JMHrapBGxcZriUw852.NCqwFRpZJFvoYxq.mV0jDpKCkiHBZwpdv09XJi6eFLJUYJJ6UDZ3c1eAsucVYLdWN.SvA9M6qzujj8nY4ym03PxQlMEd2OmXJogtDDJbPhA5AZEGxA39_6QQlvaBIUBnYPALYrTl9XJN_V4q3n4BXtpoBzrIJURMIn0mW3esUCGv0NukuVQrkrNtMgqT5SUmTGi0idaBYoR_2_wv4P09lsug; _ga_8FEWB057YH=GS2.1.s1760724394$o3$g1$t1760724401$j53$l0$h0"; - let user_agent = "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"; - - let builder = client - .get(WEBSOCKET_HTTPS_URL) - .header("Host", "grok.com") - .header("Accept", "*/*") - .header("Cookie", cookies) - .header("Connection", "Upgrade") - .header("Sec-WebSocket-Version", "13") - .header("Sec-WebSocket-Key", "JpMsHjXLPEmOsa89FQBeHA==") - .header("Upgrade", "websocket") - .header("User-Agent", user_agent); - - let response = builder.send() - .await - .map_err(|err| GrokClientError::WreqClientError(err))?; - - println!("Response version: {:?}", response.version()); - - let status = response.status(); - println!("Status: {}", status); - - Ok(()) -} - - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::setup_test_logging::setup_test_logging; - use errors::AnyhowResult; - use log::LevelFilter; - - #[tokio::test] - #[ignore] // manually test - async fn create() -> AnyhowResult<()> { - setup_test_logging(LevelFilter::Trace); - let _result = create_listen_websocket_raw_non_websocket().await; - log::logger().flush(); - assert_eq!(1, 2); - Ok(()) - } -} diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/chrome_listen_detail.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/chrome_listen_detail.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/chrome_listen_detail.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/chrome_listen_detail.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/chrome_listen_request.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/chrome_listen_request.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/chrome_listen_request.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/chrome_listen_request.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/chrome_listen_response.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/chrome_listen_response.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/chrome_listen_response.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/chrome_listen_response.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/firefox_listen_detail.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/firefox_listen_detail.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/firefox_listen_detail.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/firefox_listen_detail.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/firefox_listen_request.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/firefox_listen_request.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/firefox_listen_request.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/firefox_listen_request.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/firefox_listen_response.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/firefox_listen_response.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/firefox_listen_response.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/firefox_listen_response.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/notes.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/notes.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/notes.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/notes.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_detail.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_detail.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_detail.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_detail.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_2.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request_2.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_2.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request_2.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_3.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request_3.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_3.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request_3.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_raw_1.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request_raw_1.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_raw_1.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_request_raw_1.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_response.txt b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_response.txt similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_response.txt rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/mitm_proxy/wreq_listen_response.txt diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/tls_fingerprint_checking.rs b/crates/api_clients/grok_client/src/requests/listen_websocket/metadata/tls_fingerprint_checking.rs similarity index 100% rename from crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/tls_fingerprint_checking.rs rename to crates/api_clients/grok_client/src/requests/listen_websocket/metadata/tls_fingerprint_checking.rs diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/mod.rs b/crates/api_clients/grok_client/src/requests/listen_websocket/mod.rs index 6220fab954..3a3184d5c2 100644 --- a/crates/api_clients/grok_client/src/requests/listen_websocket/mod.rs +++ b/crates/api_clients/grok_client/src/requests/listen_websocket/mod.rs @@ -1,6 +1,4 @@ pub (crate) mod cookies; -mod create_listen_websocket; -mod create_listen_websocket_new; -mod create_listen_websocket_new_raw; -mod create_listen_websocket_raw_non_websocket; -mod tls_check; +pub mod create_listen_websocket; +pub mod create_listen_websocket_new; +pub mod create_listen_websocket_new_raw; diff --git a/crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/mod.rs b/crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/mod.rs deleted file mode 100644 index 058eb7fa3c..0000000000 --- a/crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod tls_fingerprint_checking; \ No newline at end of file diff --git a/crates/lib/cloudflare_mitigation/Cargo.toml b/crates/lib/cloudflare_mitigation/Cargo.toml new file mode 100644 index 0000000000..03da787814 --- /dev/null +++ b/crates/lib/cloudflare_mitigation/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "cloudflare_mitigation" +edition = "2024" +version = "0.0.1" +publish = false + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-build-acceleration"] + +[lib] +name = "cloudflare_mitigation" +path = "src/lib.rs" + +[dependencies] +# cargo-hakari workspace hack (faster compile times) +workspace-build-acceleration.workspace = true + +# Internal +# (None) + +# External +once_cell.workspace = true +wreq.workspace = true + +[dev-dependencies] +# (None) diff --git a/crates/lib/cloudflare_mitigation/src/headers/firefox_websocket_http_1_1_headers.rs b/crates/lib/cloudflare_mitigation/src/headers/firefox_websocket_http_1_1_headers.rs new file mode 100644 index 0000000000..3502b9fe06 --- /dev/null +++ b/crates/lib/cloudflare_mitigation/src/headers/firefox_websocket_http_1_1_headers.rs @@ -0,0 +1,33 @@ +use once_cell::sync::Lazy; +use wreq::header::OrigHeaderMap; + +/// Header casing and order needed to bypass Cloudflare WebSocket HTTP/1.1 checks. +/// This is the set of headers sent by Firefox 143 on macOS. +static FIREFOX_WEBSOCKET_HTTP_1_1_HEADERS : Lazy = Lazy::new(|| { + let mut original_headers = OrigHeaderMap::new(); + + original_headers.insert("Host"); + original_headers.insert("User-Agent"); + original_headers.insert("Accept"); + original_headers.insert("Accept-Language"); + original_headers.insert("Accept-Encoding"); + original_headers.insert("Sec-WebSocket-Version"); + original_headers.insert("Origin"); + original_headers.insert("Sec-WebSocket-Extensions"); + original_headers.insert("Sec-WebSocket-Key"); + original_headers.insert("Connection"); + original_headers.insert("Cookie"); + original_headers.insert("Sec-Fetch-Dest"); + original_headers.insert("Sec-Fetch-Mode"); + original_headers.insert("Sec-Fetch-Site"); + original_headers.insert("Pragma"); + original_headers.insert("Cache-Control"); + original_headers.insert("Upgrade"); + + original_headers +}); + +/// Wreq expects an owned copy, so we are required to clone. +pub fn get_firefox_websocket_http_1_1_headers() -> OrigHeaderMap { + FIREFOX_WEBSOCKET_HTTP_1_1_HEADERS.clone() +} \ No newline at end of file diff --git a/crates/lib/cloudflare_mitigation/src/headers/mod.rs b/crates/lib/cloudflare_mitigation/src/headers/mod.rs new file mode 100644 index 0000000000..6352f64f1d --- /dev/null +++ b/crates/lib/cloudflare_mitigation/src/headers/mod.rs @@ -0,0 +1 @@ +pub mod firefox_websocket_http_1_1_headers; \ No newline at end of file diff --git a/crates/lib/cloudflare_mitigation/src/lib.rs b/crates/lib/cloudflare_mitigation/src/lib.rs new file mode 100644 index 0000000000..a97581c44d --- /dev/null +++ b/crates/lib/cloudflare_mitigation/src/lib.rs @@ -0,0 +1 @@ +pub mod headers; \ No newline at end of file