From 5a51da5223a5116e7ca5bc4ac9cf993662afef39 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 14 Aug 2025 15:08:00 -0400 Subject: [PATCH] better endpoint - user id, email, and websocket token --- Cargo.lock | 158 ++++++++++++++++-- .../api_clients/midjourney_client/Cargo.toml | 3 + .../src/credentials/midjourney_user_id.rs | 14 ++ .../src/endpoints/get_index_page_html.rs | 95 +++++++++++ .../midjourney_client/src/endpoints/mod.rs | 1 + .../src/error/midjourney_api_error.rs | 10 ++ .../src/recipes/get_user_info.rs | 153 +++++++++++++++++ .../midjourney_client/src/recipes/mod.rs | 3 +- 8 files changed, 423 insertions(+), 14 deletions(-) create mode 100644 crates/api_clients/midjourney_client/src/endpoints/get_index_page_html.rs create mode 100644 crates/api_clients/midjourney_client/src/recipes/get_user_info.rs diff --git a/Cargo.lock b/Cargo.lock index 88a33eabfc..72b56ec851 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1261,6 +1261,21 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e0d60973d9320722cb1206f412740e162a33b8547ea8d6be75d7cff237c7a85" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit_field" version = "0.10.2" @@ -2481,6 +2496,19 @@ dependencies = [ "smallvec 1.13.2", ] +[[package]] +name = "cssparser" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e901edd733a1472f944a45116df3f846f54d37e67e68640ac8bb69689aca2aa" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa 1.0.11", + "phf 0.11.2", + "smallvec 1.13.2", +] + [[package]] name = "cssparser-macros" version = "0.6.1" @@ -3091,6 +3119,21 @@ dependencies = [ "litrs", ] +[[package]] +name = "dom_query" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2646c2bb4993c86006961293208b69d86a2b254356889ddad826e9df3f684f9" +dependencies = [ + "bit-set", + "cssparser 0.35.0", + "foldhash", + "html5ever 0.35.0", + "precomputed-hash", + "selectors 0.30.0", + "tendril", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -3824,6 +3867,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -4690,12 +4739,23 @@ checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" dependencies = [ "log", "mac", - "markup5ever", + "markup5ever 0.11.0", "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "html5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4" +dependencies = [ + "log", + "markup5ever 0.35.0", + "match_token", +] + [[package]] name = "http" version = "0.2.11" @@ -5686,7 +5746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f29e4755b7b995046f510a7520c42b2fed58b77bd94d5a87a8eb43d2fd126da8" dependencies = [ "cssparser 0.27.2", - "html5ever", + "html5ever 0.26.0", "indexmap 1.9.3", "matches", "selectors 0.22.0", @@ -6068,12 +6128,34 @@ dependencies = [ "tendril", ] +[[package]] +name = "markup5ever" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "311fe69c934650f8f19652b3946075f0fc41ad8757dbb68f1ca14e7900ecc1c3" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + [[package]] name = "match_cfg" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "match_token" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "matches" version = "0.1.10" @@ -6198,6 +6280,7 @@ dependencies = [ "anyhow", "cloudflare_errors", "cookie_store 0.0.1", + "dom_query", "errors", "filesys", "futures 0.3.31", @@ -7481,6 +7564,16 @@ dependencies = [ "phf_shared 0.10.0", ] +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.3", +] + [[package]] name = "phf_generator" version = "0.8.0" @@ -9131,7 +9224,7 @@ dependencies = [ "cssparser 0.31.2", "ego-tree", "getopts", - "html5ever", + "html5ever 0.26.0", "once_cell", "selectors 0.25.0", "smallvec 1.13.2", @@ -9236,6 +9329,25 @@ dependencies = [ "smallvec 1.13.2", ] +[[package]] +name = "selectors" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df44ba8a7ca7a4d28c589e04f526266ed76b6cc556e33fe69fa25de31939a65" +dependencies = [ + "bitflags 2.5.0", + "cssparser 0.35.0", + "derive_more 2.0.1", + "fxhash", + "log", + "new_debug_unreachable", + "phf 0.11.2", + "phf_codegen 0.11.3", + "precomputed-hash", + "servo_arc 0.4.1", + "smallvec 1.13.2", +] + [[package]] name = "semver" version = "0.9.0" @@ -9526,6 +9638,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "servo_arc" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "204ea332803bd95a0b60388590d59cf6468ec9becf626e2451f1d26a1d972de4" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.6.1" @@ -10243,26 +10364,25 @@ dependencies = [ [[package]] name = "string_cache" -version = "0.8.7" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", - "once_cell", "parking_lot 0.12.2", - "phf_shared 0.10.0", + "phf_shared 0.11.3", "precomputed-hash", "serde", ] [[package]] name = "string_cache_codegen" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", + "phf_generator 0.11.2", + "phf_shared 0.11.3", "proc-macro2", "quote", ] @@ -10991,7 +11111,7 @@ dependencies = [ "ctor", "dunce", "glob", - "html5ever", + "html5ever 0.26.0", "http 1.1.0", "infer 0.19.0", "json-patch", @@ -12472,6 +12592,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414" +dependencies = [ + "phf 0.11.2", + "phf_codegen 0.11.3", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "web_base64" version = "0.0.1" @@ -13385,7 +13517,7 @@ dependencies = [ "dunce", "gdkx11", "gtk", - "html5ever", + "html5ever 0.26.0", "http 1.1.0", "javascriptcore-rs", "jni", diff --git a/crates/api_clients/midjourney_client/Cargo.toml b/crates/api_clients/midjourney_client/Cargo.toml index 5b134c45d5..b1e29a729e 100644 --- a/crates/api_clients/midjourney_client/Cargo.toml +++ b/crates/api_clients/midjourney_client/Cargo.toml @@ -38,6 +38,9 @@ wreq = { version = "5", features = ["json"] } wreq-util = "2" tokio = { version = "1", features = ["full"] } +# NB: We need to parse HTML to get out the websocket token +dom_query = "0.20.2" + [dev-dependencies] filesys.workspace = true test_data.workspace = true diff --git a/crates/api_clients/midjourney_client/src/credentials/midjourney_user_id.rs b/crates/api_clients/midjourney_client/src/credentials/midjourney_user_id.rs index 94b74b9365..55a7d4bbb2 100644 --- a/crates/api_clients/midjourney_client/src/credentials/midjourney_user_id.rs +++ b/crates/api_clients/midjourney_client/src/credentials/midjourney_user_id.rs @@ -2,3 +2,17 @@ /// The user ID used for several midjourney requests. #[derive(Clone, Debug, Eq, PartialEq, Hash)] pub struct MidjourneyUserId(pub String); + +impl MidjourneyUserId { + pub fn from_str(user_id: &str) -> Self { + Self(user_id.to_string()) + } + + pub fn from_string(user_id: String) -> Self { + Self(user_id) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} diff --git a/crates/api_clients/midjourney_client/src/endpoints/get_index_page_html.rs b/crates/api_clients/midjourney_client/src/endpoints/get_index_page_html.rs new file mode 100644 index 0000000000..90949efa9d --- /dev/null +++ b/crates/api_clients/midjourney_client/src/endpoints/get_index_page_html.rs @@ -0,0 +1,95 @@ +use crate::client::midjourney_hostname::MidjourneyHostname; +use crate::error::midjourney_api_error::MidjourneyApiError; +use crate::error::midjourney_client_error::MidjourneyClientError; +use crate::error::midjourney_error::MidjourneyError; +use cloudflare_errors::filter_cloudflare_errors::filter_cloudflare_errors; +use log::error; +use serde::Deserialize; +use wreq::Client; +use wreq_util::Emulation; + +/// This grabs the pre-rendered HTML page, which contains the websocket token and other details. +pub struct GetIndexPageRequest { + pub hostname: MidjourneyHostname, + pub cookie_header: String, +} + +pub async fn get_index_page_html(req: GetIndexPageRequest) -> Result { + let url = format!("https://{}/", req.hostname.as_str()); + + let client = Client::builder() + .emulation(Emulation::Firefox139) + .build() + .map_err(|err| MidjourneyClientError::WreqError(err))?; + + let cookie_header = req.cookie_header.trim(); + + if cookie_header.len() < 20 { + error!("Cookie header is too short (len: {}): {}", cookie_header.len(), cookie_header); + return Err(MidjourneyClientError::CookieTooShort.into()); + } + + // NB: missing headers that were in the browser request: + // -H 'sec-ch-ua-platform: "macOS"' \ + // -H 'sec-ch-ua: "Not)A;Brand";v="8", "Chromium";v="138", "Google Chrome";v="138"' \ + + let mut http_request = client.get(url) + .header("cookie", cookie_header) + .header("Referrer-Policy", "origin-when-cross-origin") + .header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + .header("accept-language", "en-US,en;q=0.8") + .header("content-type", "application/json") + .header("priority", "u=0, i") + .header("sec-ch-ua-mobile", "?0") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") + .header("sec-fetch-site", "none") + .header("sec-fetch-user", "?1") + .header("upgrade-insecure-requests", "1"); + + let http_request = http_request + .build() + .map_err(|err| MidjourneyClientError::WreqError(err))?; + + let response = client.execute(http_request) + .await + .map_err(|e| MidjourneyApiError::NetworkError(e.to_string()))?; + + let status = response.status(); + + let response_body = response.text().await + .map_err(|e| MidjourneyApiError::NetworkError(e.to_string()))?; + + if !status.is_success() { + if let Err(err) = filter_cloudflare_errors(status.as_u16(), &response_body) { + return Err(MidjourneyApiError::CloudflareError(err).into()); + } + } + + Ok(response_body) +} + +#[cfg(test)] +mod tests { + use crate::client::midjourney_hostname::MidjourneyHostname; + use errors::AnyhowResult; + use filesys::read_to_trimmed_string::read_to_trimmed_string; + use crate::endpoints::get_index_page_html::{get_index_page_html, GetIndexPageRequest}; + + #[ignore] + #[tokio::test] + async fn test() -> AnyhowResult<()> { + let cookie_header = read_to_trimmed_string("/Users/bt/secrets/midjourney/cookie.txt")?; + + let result = get_index_page_html(GetIndexPageRequest { + cookie_header, + hostname: MidjourneyHostname::Standard, + }).await?; + + println!("Response: {:?}\n\n", result); + + assert_eq!(1, 2); + + Ok(()) + } +} diff --git a/crates/api_clients/midjourney_client/src/endpoints/mod.rs b/crates/api_clients/midjourney_client/src/endpoints/mod.rs index de92b74c8a..78717387ce 100644 --- a/crates/api_clients/midjourney_client/src/endpoints/mod.rs +++ b/crates/api_clients/midjourney_client/src/endpoints/mod.rs @@ -1,3 +1,4 @@ +pub mod get_index_page_html; pub mod storage_list; pub mod submit_job; pub mod user_connections; \ No newline at end of file diff --git a/crates/api_clients/midjourney_client/src/error/midjourney_api_error.rs b/crates/api_clients/midjourney_client/src/error/midjourney_api_error.rs index 6de7fe205f..2c417d5712 100644 --- a/crates/api_clients/midjourney_client/src/error/midjourney_api_error.rs +++ b/crates/api_clients/midjourney_client/src/error/midjourney_api_error.rs @@ -12,6 +12,14 @@ pub enum MidjourneyApiError { /// There was no user ID in an otherwise valid-looking response. NoUserId, + /// No user props in index HTML payload. + /// We need this for the user ID and the websocket token. + NoUserProps, + + /// No initialAuthUser in the index HTML payload. + /// We need this for the user ID and the websocket token. + NoInitialAuthUser, + /// 400. The request was invalid. InvalidRequest(String), @@ -63,6 +71,8 @@ impl Display for MidjourneyApiError { // Response body errors Self::NoJobId => write!(f, "No job ID found in the response body."), Self::NoUserId => write!(f, "No user ID found in the response body."), + Self::NoUserProps => write!(f, "No user properties found in the index HTML payload."), + Self::NoInitialAuthUser => write!(f, "No initialAuthUser in the index HTML payload."), // Server response code errors Self::InvalidRequest(msg) => write!(f, "Invalid request: {}", msg), Self::Unauthorized(msg) => write!(f, "Unauthorized: {}", msg), diff --git a/crates/api_clients/midjourney_client/src/recipes/get_user_info.rs b/crates/api_clients/midjourney_client/src/recipes/get_user_info.rs new file mode 100644 index 0000000000..f1fee8f084 --- /dev/null +++ b/crates/api_clients/midjourney_client/src/recipes/get_user_info.rs @@ -0,0 +1,153 @@ +use crate::client::midjourney_hostname::MidjourneyHostname; +use crate::credentials::midjourney_user_id::MidjourneyUserId; +use crate::endpoints::get_index_page_html::{get_index_page_html, GetIndexPageRequest}; +use crate::error::midjourney_api_error::MidjourneyApiError; +use crate::error::midjourney_error::MidjourneyError; +use dom_query::Document; +use serde::Deserialize; + +pub struct GetUserInfoRequest { + pub hostname: MidjourneyHostname, + pub cookie_header: String, +} + +#[derive(Debug, Clone)] +pub struct GetUserInfoResponse { + pub user_id: Option, + pub email: Option, + pub websocket_token: Option, +} + +pub async fn get_user_info(req: GetUserInfoRequest) -> Result { + let index_html = get_index_page_html(GetIndexPageRequest { + hostname: req.hostname, + cookie_header: req.cookie_header, + }).await?; + + /* + + */ + + let document = Document::from(index_html); + let maybe_nodes = document.select("script[id=initialProps]"); + + let node = match maybe_nodes.get(0) { + Some(node) => node, + None => { + return Err(MidjourneyApiError::NoUserProps.into()); + } + }; + + let inner_json = node.inner_html().to_string(); + + #[derive(Deserialize, Debug)] + #[allow(non_snake_case)] + struct RawAuthUser { + // NB: The Midjourney user id. + id: Option, + email: Option, + websocketToken: Option, + } + + #[derive(Deserialize, Debug)] + #[allow(non_snake_case)] + struct RawBody { + initialAuthUser: Option, + } + + let response : RawBody = serde_json::from_str(&inner_json) + .map_err(|err| MidjourneyApiError::DeserializationError(err))?; + + let props = match response.initialAuthUser { + Some(props) => props, + None => { + return Err(MidjourneyApiError::NoInitialAuthUser.into()); + } + }; + + Ok(GetUserInfoResponse { + user_id: props.id + .map(|id| MidjourneyUserId::from_string(id)), + email: props.email + .map(|email| email.to_string()), + websocket_token: props.websocketToken + .map(|email| email.to_string()), + }) +} + +#[cfg(test)] +mod tests { + use crate::client::midjourney_hostname::MidjourneyHostname; + use crate::recipes::get_user_info::{get_user_info, GetUserInfoRequest}; + use errors::AnyhowResult; + use filesys::read_to_trimmed_string::read_to_trimmed_string; + + #[ignore] + #[tokio::test] + async fn test() -> AnyhowResult<()> { + let cookie_header = read_to_trimmed_string("/Users/bt/secrets/midjourney/cookie.txt")?; + + let result = get_user_info(GetUserInfoRequest { + cookie_header, + hostname: MidjourneyHostname::Standard, + }).await?; + + println!("Response: {:?}\n\n", result); + + assert_eq!(1, 2); + + Ok(()) + } +} + diff --git a/crates/api_clients/midjourney_client/src/recipes/mod.rs b/crates/api_clients/midjourney_client/src/recipes/mod.rs index 8c812ba667..04f4ba3241 100644 --- a/crates/api_clients/midjourney_client/src/recipes/mod.rs +++ b/crates/api_clients/midjourney_client/src/recipes/mod.rs @@ -1,2 +1,3 @@ mod get_midjourney_user_id; -mod channel_id; \ No newline at end of file +mod channel_id; +mod get_user_info; \ No newline at end of file