cleanup, new lib

Cargo.lock

@@ -2180,6 +2180,15 @@ dependencies = [
  "workspace-build-acceleration",
 ]
 
+[[package]]
+name = "cloudflare_mitigation"
+version = "0.0.1"
+dependencies = [
+ "once_cell",
+ "workspace-build-acceleration",
+ "wreq",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.54"
@@ -4666,6 +4675,7 @@ dependencies = [
  "anyhow",
  "chrono",
  "cloudflare_errors",
+ "cloudflare_mitigation",
  "cookie_store 0.0.1",
  "env",
  "env_logger",

Cargo.toml

@@ -44,6 +44,7 @@ members = [
   "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
   "crates/lib/cloudflare_errors",
+  "crates/lib/cloudflare_mitigation",
   "crates/lib/concurrency",
   "crates/lib/cookie_store",
   "crates/lib/cookies",
@@ -187,6 +188,7 @@ artcraft_api_defs = { path = "crates/api_clients/artcraft/artcraft_api_defs" }
 build_metadata = { path = "crates/lib/build_metadata" }
 cloud_storage = { path = "crates/api_clients/cloud_storage" }
 cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
+cloudflare_mitigation = { path = "crates/lib/cloudflare_mitigation" }
 component_traits = { path = "crates/service/plugins/component_traits" }
 cookie_store = { path = "crates/lib/cookie_store" }
 crockford = { path = "crates/lib/data/crockford" }

crates/api_clients/grok_client/Cargo.toml

@@ -17,6 +17,7 @@ workspace-build-acceleration.workspace = true
 
 # Internal
 cloudflare_errors.workspace = true
+cloudflare_mitigation.workspace = true
 cookie_store.workspace = true
 errors.workspace = true
 

crates/api_clients/grok_client/src/error/grok_generic_api_error.rs

@@ -23,6 +23,9 @@ pub enum GrokGenericApiError {
     body: String,
   },
 
+  /// An error upgrading the connection to a websocket.
+  WreqWebsocketUpgradeError(wreq::Error),
+
   /// An uncaught error from the API client.
   WreqError(wreq::Error),
 }
@@ -36,6 +39,7 @@ impl Display for GrokGenericApiError {
       Self::SerdeResponseParseErrorWithBody(err, body) => write!(f, "Failed to parse response body: {:?}. Body: {}", err, body),
       Self::SerdeParseErrorWithBodyOnNon200(err, body) => write!(f, "Failed to parse non-200 response body: {:?}. Body: {}", err, body),
       Self::UncategorizedBadResponseWithStatusAndBody { status_code, body } => write!(f, "Uncategorized bad response: status code {}, body: {}", status_code, body),
+      Self::WreqWebsocketUpgradeError(err) => write!(f, "Websocket upgrade error: {}", err),
       Self::WreqError(err) => write!(f, "Wreq client error: {}", err),
     }
   }

crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_new.rs

@@ -1,55 +1,32 @@
+use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
 use crate::error::grok_client_error::GrokClientError;
 use crate::error::grok_error::GrokError;
+use crate::error::grok_generic_api_error::GrokGenericApiError;
 use crate::requests::listen_websocket::cookies::SESSION_COOKIES_WITHOUT_CF_CLEARANCE;
+use cloudflare_mitigation::headers::firefox_websocket_http_1_1_headers::get_firefox_websocket_http_1_1_headers;
 use std::ops::Deref;
 use std::time::Duration;
+use log::info;
 use tokio::io::AsyncReadExt;
-use wreq::{Client, Proxy};
 use wreq::header::{ACCEPT, ACCEPT_LANGUAGE, CACHE_CONTROL, COOKIE, ORIGIN, PRAGMA, SEC_WEBSOCKET_EXTENSIONS, USER_AGENT};
+use wreq::{Client, Proxy};
 use wreq_util::Emulation;
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
 
 const WEBSOCKET_URL: &str = "wss://grok.com/ws/imagine/listen";
 
 pub async fn create_listen_websocket_new() -> Result<(), GrokError> {
-  /*
-  Wreq Firefox vs Real Firefox Private Mode:
-
-   - wreq has two extensions real firefox doesn't:
-      - "session_ticket (35) (IANA)"
-      - "psk_key_exchange_modes (45) (IANA)"
-   - wreq has "psk_key_exchange_mode": "AQ==", whereas real firefox private mode is null
-   - wreq has "session_ticket_supported": true, whereas real firefox private mode is false
-
-  Wreq Firefox is IDENTICAL to Real Firefox Normal Mode (non-private) mode. - So why is it not working!?
-   */
-
   let mut client_builder = Client::builder()
       .emulation(Emulation::Firefox143)
       .connection_verbose(true)
       .connect_timeout(Duration::from_secs(10));
 
-  let mut proxy = None;
-
-  proxy = Some(Proxy::https("http://127.0.0.1:8080")
-      .map_err(|err| GrokClientError::WreqClientError(err))?);
-
-  if let Some(proxy) = proxy {
-    client_builder = client_builder
-        .cert_verification(false)
-        .proxy(proxy);
-  }
-
   let client = client_builder
       .build()
       .map_err(|err| GrokClientError::WreqClientError(err))?;
 
-  //.http1_only() // NB: Not needed - websockets are sent over HTTP/1.1 without this configuration
-  //.cookie_store(true)
-
   let builder = client.websocket(WEBSOCKET_URL)
-      //.header("Host", "grok.com")
       .default_headers(false)
+      .orig_headers(get_firefox_websocket_http_1_1_headers())
       .header(COOKIE, SESSION_COOKIES_WITHOUT_CF_CLEARANCE)
       .header(ORIGIN,"https://grok.com")
       .header(USER_AGENT, FIREFOX_143_MAC_USER_AGENT)
@@ -60,29 +37,25 @@ pub async fn create_listen_websocket_new() -> Result<(), GrokError> {
       .header("Sec-Fetch-Mode", "websocket")
       .header("Sec-Fetch-Site", "same-origin")
       .header(PRAGMA, "no-cache")
-      .header(CACHE_CONTROL, "no-cache")
-      ;
-
-  // TODO: Connect: keep-alive --> keep-alive, Upgrade
-  // TODO: Accept-Encoding: "gzip, deflate, br, zstd"
-  // TODO: Camel case header names
-  // TODO: Header orders!
+      .header(CACHE_CONTROL, "no-cache");
 
   let response = builder.send()
       .await
       .map_err(|err| GrokClientError::WreqClientError(err))?;
 
-  println!("Response version: {:?}", response.version());
-
   let status = response.status();
-  println!("Status: {}", status);
 
-  for cookie in response.cookies() {
-    println!("Cookie: {}={}", cookie.name(), cookie.value());
+  info!("WebSocket Response Status: {}", status);
+
+  if !status.is_success() || status.as_u16() != 101 {
   }
 
-  for (name, value) in response.headers() {
-    println!("Header: {}: {}", name.as_str(), value.to_str().unwrap());
+  let mut websocket = response.into_websocket()
+      .await
+      .map_err(|err| GrokGenericApiError::WreqWebsocketUpgradeError(err))?;
+
+  if let Some(protocol) = websocket.protocol() {
+    println!("WebSocket subprotocol: {:?}", protocol);
   }
 
   Ok(())

crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_raw_non_websocket.rs

@@ -1,67 +0,0 @@
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use std::ops::Deref;
-use std::time::Duration;
-use tokio::io::AsyncReadExt;
-use wreq::Client;
-use wreq_util::Emulation;
-
-const WEBSOCKET_WSS_URL: &str = "wss://grok.com/ws/imagine/listen";
-const WEBSOCKET_HTTPS_URL: &str = "https://grok.com/ws/imagine/listen";
-
-pub async fn create_listen_websocket_raw_non_websocket() -> Result<(), GrokError> {
-  let client = Client::builder()
-      .emulation(Emulation::Chrome140)
-      .http1_only()
-      .connection_verbose(true)
-      .connect_timeout(Duration::from_secs(10))
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-  
-  //.http1_only() // NB: Not needed - websockets are sent over HTTP/1.1 without this configuration
-  //.cookie_store(true)
-
-  let cookies = "_ga=GA1.1.1232202746.1760710013; i18nextLng=en; sso=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzZXNzaW9uX2lkIjoiOGU3MDFiNzctOTdkNC00ZjM0LWExOTctOWFmMDU1MzY3NDAwIn0.-a6x0InxbGzfTVfUlrdzxskxCnvMDI8lC90z4wHeGIk; sso-rw=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzZXNzaW9uX2lkIjoiOGU3MDFiNzctOTdkNC00ZjM0LWExOTctOWFmMDU1MzY3NDAwIn0.-a6x0InxbGzfTVfUlrdzxskxCnvMDI8lC90z4wHeGIk; stblid=b3331fc1-45d7-466b-83df-67427c0b2367; mp_ea93da913ddb66b6372b89d97b1029ac_mixpanel=%7B%22distinct_id%22%3A%2285980643-ffab-4984-a3de-59a608c47d7f%22%2C%22%24device_id%22%3A%2279ce237a-a0f3-4913-bf4b-519ac8a98263%22%2C%22%24initial_referrer%22%3A%22%24direct%22%2C%22%24initial_referring_domain%22%3A%22%24direct%22%2C%22__mps%22%3A%7B%7D%2C%22__mpso%22%3A%7B%7D%2C%22__mpus%22%3A%7B%7D%2C%22__mpa%22%3A%7B%7D%2C%22__mpu%22%3A%7B%7D%2C%22__mpr%22%3A%5B%5D%2C%22__mpap%22%3A%5B%5D%2C%22%24user_id%22%3A%2285980643-ffab-4984-a3de-59a608c47d7f%22%7D; cf_clearance=e7.LXFbmc.U81rZIqxuBfBe88yuBZMXEP.zwxNkbOxw-1760724394-1.2.1.1-JMHrapBGxcZriUw852.NCqwFRpZJFvoYxq.mV0jDpKCkiHBZwpdv09XJi6eFLJUYJJ6UDZ3c1eAsucVYLdWN.SvA9M6qzujj8nY4ym03PxQlMEd2OmXJogtDDJbPhA5AZEGxA39_6QQlvaBIUBnYPALYrTl9XJN_V4q3n4BXtpoBzrIJURMIn0mW3esUCGv0NukuVQrkrNtMgqT5SUmTGi0idaBYoR_2_wv4P09lsug; _ga_8FEWB057YH=GS2.1.s1760724394$o3$g1$t1760724401$j53$l0$h0";
-  let user_agent = "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
-
-  let builder = client
-      .get(WEBSOCKET_HTTPS_URL)
-      .header("Host", "grok.com")
-      .header("Accept", "*/*")
-      .header("Cookie", cookies)
-      .header("Connection", "Upgrade")
-      .header("Sec-WebSocket-Version", "13")
-      .header("Sec-WebSocket-Key", "JpMsHjXLPEmOsa89FQBeHA==")
-      .header("Upgrade", "websocket")
-      .header("User-Agent", user_agent);
-
-  let response = builder.send()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  println!("Response version: {:?}", response.version());
-
-  let status = response.status();
-  println!("Status: {}", status);
-  
-  Ok(())
-}
-
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn create() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Trace);
-    let _result = create_listen_websocket_raw_non_websocket().await;
-    log::logger().flush();
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_client/src/requests/listen_websocket/mod.rs

@@ -1,6 +1,4 @@
 pub (crate) mod cookies;
-mod create_listen_websocket;
-mod create_listen_websocket_new;
-mod create_listen_websocket_new_raw;
-mod create_listen_websocket_raw_non_websocket;
-mod tls_check;
+pub mod create_listen_websocket;
+pub mod create_listen_websocket_new;
+pub mod create_listen_websocket_new_raw;

crates/api_clients/grok_client/src/requests/listen_websocket/tls_check/mod.rs

@@ -1 +0,0 @@
-mod tls_fingerprint_checking;

crates/lib/cloudflare_mitigation/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "cloudflare_mitigation"
+edition = "2024"
+version = "0.0.1"
+publish = false
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-build-acceleration"]
+
+[lib]
+name = "cloudflare_mitigation"
+path = "src/lib.rs"
+
+[dependencies]
+# cargo-hakari workspace hack (faster compile times)
+workspace-build-acceleration.workspace = true
+
+# Internal
+# (None)
+
+# External
+once_cell.workspace = true
+wreq.workspace = true
+
+[dev-dependencies]
+# (None)

crates/lib/cloudflare_mitigation/src/headers/firefox_websocket_http_1_1_headers.rs

@@ -0,0 +1,33 @@
+use once_cell::sync::Lazy;
+use wreq::header::OrigHeaderMap;
+
+/// Header casing and order needed to bypass Cloudflare WebSocket HTTP/1.1 checks.
+/// This is the set of headers sent by Firefox 143 on macOS.
+static FIREFOX_WEBSOCKET_HTTP_1_1_HEADERS : Lazy<OrigHeaderMap> = Lazy::new(|| {
+  let mut original_headers = OrigHeaderMap::new();
+
+  original_headers.insert("Host");
+  original_headers.insert("User-Agent");
+  original_headers.insert("Accept");
+  original_headers.insert("Accept-Language");
+  original_headers.insert("Accept-Encoding");
+  original_headers.insert("Sec-WebSocket-Version");
+  original_headers.insert("Origin");
+  original_headers.insert("Sec-WebSocket-Extensions");
+  original_headers.insert("Sec-WebSocket-Key");
+  original_headers.insert("Connection");
+  original_headers.insert("Cookie");
+  original_headers.insert("Sec-Fetch-Dest");
+  original_headers.insert("Sec-Fetch-Mode");
+  original_headers.insert("Sec-Fetch-Site");
+  original_headers.insert("Pragma");
+  original_headers.insert("Cache-Control");
+  original_headers.insert("Upgrade");
+
+  original_headers
+});
+
+/// Wreq expects an owned copy, so we are required to clone.
+pub fn get_firefox_websocket_http_1_1_headers() -> OrigHeaderMap {
+  FIREFOX_WEBSOCKET_HTTP_1_1_HEADERS.clone()
+}

crates/lib/cloudflare_mitigation/src/headers/mod.rs

@@ -0,0 +1 @@
+pub mod firefox_websocket_http_1_1_headers;

crates/lib/cloudflare_mitigation/src/lib.rs

@@ -0,0 +1 @@
+pub mod headers;