wip

crates/api_clients/grok_client/src/requests/listen_websocket/create_listen_websocket_new_raw.rs

@@ -0,0 +1,117 @@
+use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
+use crate::error::grok_client_error::GrokClientError;
+use crate::error::grok_error::GrokError;
+use crate::requests::listen_websocket::cookies::SESSION_COOKIES_WITHOUT_CF_CLEARANCE;
+use std::ops::Deref;
+use std::time::Duration;
+use tokio::io::AsyncReadExt;
+use wreq::header::{HeaderMap, ACCEPT, ACCEPT_LANGUAGE, CACHE_CONTROL, CONNECTION, COOKIE, HOST, ORIGIN, PRAGMA, SEC_WEBSOCKET_EXTENSIONS, SEC_WEBSOCKET_KEY, SEC_WEBSOCKET_VERSION, UPGRADE, USER_AGENT};
+use wreq::{Client, Proxy, Version};
+use wreq_util::Emulation;
+
+const WEBSOCKET_HTTPS_URL: &str = "https://grok.com/ws/imagine/listen";
+
+pub async fn create_listen_websocket_new_raw() -> Result<(), GrokError> {
+  /*
+  Wreq Firefox vs Real Firefox Private Mode:
+
+   - wreq has two extensions real firefox doesn't:
+      - "session_ticket (35) (IANA)"
+      - "psk_key_exchange_modes (45) (IANA)"
+   - wreq has "psk_key_exchange_mode": "AQ==", whereas real firefox private mode is null
+   - wreq has "session_ticket_supported": true, whereas real firefox private mode is false
+
+  Wreq Firefox is IDENTICAL to Real Firefox Normal Mode (non-private) mode. - So why is it not working!?
+   */
+
+  let mut client_builder = Client::builder()
+      .emulation(Emulation::Firefox143)
+      .default_headers(HeaderMap::new())
+      .connection_verbose(true)
+      .connect_timeout(Duration::from_secs(10));
+
+  let mut proxy = None;
+
+  proxy = Some(Proxy::https("http://127.0.0.1:8080")
+      .map_err(|err| GrokClientError::WreqClientError(err))?);
+
+  if let Some(proxy) = proxy {
+    client_builder = client_builder
+        .cert_verification(false)
+        .proxy(proxy);
+  }
+
+  let client = client_builder
+      .build()
+      .map_err(|err| GrokClientError::WreqClientError(err))?;
+
+  //.http1_only() // NB: Not needed - websockets are sent over HTTP/1.1 without this configuration
+  //.cookie_store(true)
+
+  // NB: Sending a normal GET request seems to punch through Cloudflare and get a 400 instead
+  let builder = client.get(WEBSOCKET_HTTPS_URL)
+      .version(Version::HTTP_11)
+      .default_headers(false)
+      .header(HOST, "grok.com")
+      .header(COOKIE, SESSION_COOKIES_WITHOUT_CF_CLEARANCE)
+      .header(ORIGIN,"https://grok.com")
+      .header(USER_AGENT, FIREFOX_143_MAC_USER_AGENT)
+      .header(SEC_WEBSOCKET_EXTENSIONS, "permessage-deflate")
+      .header(ACCEPT, "*/*")
+      .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
+      .header("Sec-Fetch-Dest", "empty")
+      .header("Sec-Fetch-Mode", "websocket")
+      .header("Sec-Fetch-Site", "same-origin")
+      .header(SEC_WEBSOCKET_VERSION, 13)
+      .header(SEC_WEBSOCKET_KEY, "X2NHDjwqbk4quToBT5L97Q==")
+      .header(CONNECTION, "keep-alive, Upgrade")
+      .header(PRAGMA, "no-cache")
+      .header(CACHE_CONTROL, "no-cache")
+      .header(UPGRADE, "websocket")
+      ;
+
+  // TODO: Connect: keep-alive --> keep-alive, Upgrade
+  // TODO: Accept-Encoding: "gzip, deflate, br, zstd"
+  // TODO: Camel case header names
+  // TODO: Header orders!
+
+  let response = builder.send()
+      .await
+      .map_err(|err| GrokClientError::WreqClientError(err))?;
+
+  println!("Response version: {:?}", response.version());
+
+  let status = response.status();
+  println!("Status: {}", status);
+
+  for cookie in response.cookies() {
+    println!("Cookie: {}={}", cookie.name(), cookie.value());
+  }
+
+  for (name, value) in response.headers() {
+    println!("Header: {}: {}", name.as_str(), value.to_str().unwrap());
+  }
+
+  Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use crate::test_utils::setup_test_logging::setup_test_logging;
+  use errors::AnyhowResult;
+  use log::LevelFilter;
+
+  #[tokio::test]
+  #[ignore] // manually test
+  async fn create() -> AnyhowResult<()> {
+    setup_test_logging(LevelFilter::Trace);
+    let result = create_listen_websocket_new_raw().await;
+    if let Err(err) = result {
+      println!("Error: {:?}", err);
+    }
+    log::logger().flush();
+    assert_eq!(1, 2);
+    Ok(())
+  }
+}

crates/api_clients/grok_client/src/requests/listen_websocket/mitm_proxy/wreq_listen_request_raw_1.txt

@@ -0,0 +1,20 @@
+Cookie:                    _ga=GA1.1.1232202746.1760710013; i18nextLng=en; sso=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzZXNzaW9uX2lkIjoiOGU3MDFiNzctOTdkNC00ZjM0LWExOTct
+                           OWFmMDU1MzY3NDAwIn0.-a6x0InxbGzfTVfUlrdzxskxCnvMDI8lC90z4wHeGIk; sso-rw=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzZXNzaW9uX2lkIjoiOGU3MDFiNzct
+                           OTdkNC00ZjM0LWExOTctOWFmMDU1MzY3NDAwIn0.-a6x0InxbGzfTVfUlrdzxskxCnvMDI8lC90z4wHeGIk; stblid=b3331fc1-45d7-466b-83df-67427c0b2367; mp_ea93da91
+                           3ddb66b6372b89d97b1029ac_mixpanel=%7B%22distinct_id%22%3A%2285980643-ffab-4984-a3de-59a608c47d7f%22%2C%22%24device_id%22%3A%2279ce237a-a0f3-4
+                           913-bf4b-519ac8a98263%22%2C%22%24initial_referrer%22%3A%22%24direct%22%2C%22%24initial_referring_domain%22%3A%22%24direct%22%2C%22__mps%22%3A
+                           %7B%7D%2C%22__mpso%22%3A%7B%7D%2C%22__mpus%22%3A%7B%7D%2C%22__mpa%22%3A%7B%7D%2C%22__mpu%22%3A%7B%7D%2C%22__mpr%22%3A%5B%5D%2C%22__mpap%22%3A
+                           %5B%5D%2C%22%24user_id%22%3A%2285980643-ffab-4984-a3de-59a608c47d7f%22%7D; _ga_8FEWB057YH=GS2.1.s1760724394$o3$g1$t1760724401$j53$l0$h0
+Origin:                    https://grok.com
+User-Agent:                Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0
+Sec-WebSocket-Extensions:  permessage-deflate
+Accept:                    */*
+Accept-Language:           en-US,en;q=0.5
+sec-fetch-dest:            empty
+sec-fetch-mode:            websocket
+sec-fetch-site:            same-origin
+Pragma:                    no-cache
+Cache-Control:             no-cache
+Upgrade:                   websocket
+accept-encoding:           zstd,gzip,deflate,br
+Host:                      grok.com

crates/api_clients/grok_client/src/requests/listen_websocket/mod.rs

@@ -1,5 +1,6 @@
+mod cookies;
 mod create_listen_websocket;
 mod create_listen_websocket_new;
+mod create_listen_websocket_new_raw;
 mod create_listen_websocket_raw_non_websocket;
 mod tls_check;
-mod cookies;