From bae6bc6863ecbaf161d16e60836ed746beb27575 Mon Sep 17 00:00:00 2001 From: Kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Tue, 15 Apr 2025 21:20:20 +0530 Subject: [PATCH] move sentinel token generation into rust --- Cargo.lock | 31 + Cargo.toml | 4 +- .../image/encode_rgb_image_base64_png.rs | 2 +- .../api_clients/openai_sora_client/Cargo.toml | 6 + .../src/image_gen/image_gen_http_request.rs | 10 +- .../src/sentinel_refresh/generate/config.rs | 675 ++++++++++++++++++ .../src/sentinel_refresh/generate/mod.rs | 3 + .../src/sentinel_refresh/generate/request.rs | 226 ++++++ .../src/sentinel_refresh/generate/token.rs | 68 ++ .../src/sentinel_refresh/mod.rs | 4 +- 10 files changed, 1024 insertions(+), 5 deletions(-) create mode 100644 crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/config.rs create mode 100644 crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/mod.rs create mode 100644 crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs create mode 100644 crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/token.rs diff --git a/Cargo.lock b/Cargo.lock index 2010d39b51..707ea04c4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5035,6 +5035,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-literal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcaaec4551594c969335c98c903c1397853d4198408ea609190f420500f6be71" + [[package]] name = "hf-hub" version = "0.3.2" @@ -6130,6 +6136,15 @@ dependencies = [ "signature", ] +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -7849,14 +7864,20 @@ name = "openai_sora_client" version = "0.0.1" dependencies = [ "anyhow", + "base64 0.22.1", + "chrono", "crockford", "errors", "filesys", + "hex-literal", + "idempotency", "log", + "rand 0.9.0", "reqwest 0.12.15", "serde", "serde_derive", "serde_json", + "sha3", "subprocess_common", "testing", "thiserror 2.0.11", @@ -10605,6 +10626,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest 0.10.7", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index 8e5520552e..36644b4132 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,7 +152,9 @@ workspace-build-acceleration = { path = "crates/workspace-build-acceleration" } # Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace # If a crate needs to use an older/newer version, the dependency should be managed in that crate's own Cargo.toml anyhow = "1.0.97" +base64 = "0.22.1" chrono = { version = "0.4.40", features = ["serde"] } +hex-literal = "1.0.0" log = "0.4.27" once_cell = "1.21.3" rand = "0.9.0" @@ -162,10 +164,10 @@ serde = { version = "1.0.219", features = ["derive"] } serde_derive = "1.0.219" serde_json = "1.0.140" serde_yaml = "0.9.1" +sha3 = "0.10.8" thiserror = "2" toml = "0.7.4" url = "2.5.4" - # Internal libs crockford = { path = "crates/lib/data/crockford" } errors = { path = "crates/lib/errors" } diff --git a/crates/desktop/tauri-4o/src/utils/image/encode_rgb_image_base64_png.rs b/crates/desktop/tauri-4o/src/utils/image/encode_rgb_image_base64_png.rs index fcd248eb2f..76eb333de0 100644 --- a/crates/desktop/tauri-4o/src/utils/image/encode_rgb_image_base64_png.rs +++ b/crates/desktop/tauri-4o/src/utils/image/encode_rgb_image_base64_png.rs @@ -10,7 +10,7 @@ pub fn encode_rgb_image_base64_png( image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png)?; - let bytes = BASE64_STANDARD.encode(bytes); + let bytes = (&BASE64_STANDARD).encode(bytes); Ok(bytes) } diff --git a/crates/lib/api_clients/openai_sora_client/Cargo.toml b/crates/lib/api_clients/openai_sora_client/Cargo.toml index cd2e8adde8..51e83a9626 100644 --- a/crates/lib/api_clients/openai_sora_client/Cargo.toml +++ b/crates/lib/api_clients/openai_sora_client/Cargo.toml @@ -20,6 +20,7 @@ workspace-build-acceleration = { workspace = true } crockford = { path = "../../data/crockford" } errors = { path = "../../errors" } subprocess_common = { path = "../../subprocess_common" } +idempotency = { path = "../../idempotency" } # External anyhow = { workspace = true } @@ -28,9 +29,14 @@ reqwest = { workspace = true, features = ["multipart"] } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } +sha3 = { workspace = true } +hex-literal = { workspace = true } +base64 = { workspace = true } tokio = { version = "1.38.0", features = ["macros"] } toml = { workspace = true } thiserror = { workspace = true } +chrono = { workspace = true} +rand = { workspace = true } [dev-dependencies] testing = { path = "../../testing" } diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs b/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs index 0228434833..3b6cc5541f 100644 --- a/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs @@ -15,6 +15,9 @@ pub enum SoraError { #[error("Sora sentinel block: {0}")] SentinelBlock(String), + #[error("Sora too many concurrent tasks: {0}")] + TooManyConcurrentTasks(String), + #[error("Sora API error: {0}")] GenericError(String), @@ -115,10 +118,10 @@ pub enum RawSoraError { #[derive(Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] -pub enum SoraErrorCode { +pub enum SoraErrorCode { SentinelBlock, TokenExpired, - + TooManyConcurrentTasks, Unknown(String), } @@ -179,6 +182,9 @@ pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest SoraErrorCode::SentinelBlock => { return Err(SoraError::SentinelBlock(error_response.error.message)); } + SoraErrorCode::TooManyConcurrentTasks => { + return Err(SoraError::TooManyConcurrentTasks(error_response.error.message)); + } SoraErrorCode::Unknown(_) => { return Err(SoraError::GenericError(error_response.error.message)); } diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/config.rs b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/config.rs new file mode 100644 index 0000000000..c0ff7189e8 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/config.rs @@ -0,0 +1,675 @@ +// cores = [8, 16, 24, 32] +// cached_scripts = ["https://cdn.oaistatic.com/_next/static/cXh69klOLzS0Gy2joLDRS/_ssgManifest.js?dpl=453ebaec0d44c2decab71692e1bfe39be35a24b3"] + +// cached_dpl = ["prod-f501fe933b3edf57aea882da888e1a544df99840"] + +// navigator_key = [ +// "registerProtocolHandler−function registerProtocolHandler() { [native code] }", +// "storage−[object StorageManager]", +// "locks−[object LockManager]", +// "appCodeName−Mozilla", +// "permissions−[object Permissions]", +// "share−function share() { [native code] }", +// "webdriver−false", +// "managed−[object NavigatorManagedData]", +// "canShare−function canShare() { [native code] }", +// "vendor−Google Inc.", +// "vendor−Google Inc.", +// "mediaDevices−[object MediaDevices]", +// "vibrate−function vibrate() { [native code] }", +// "storageBuckets−[object StorageBucketManager]", +// "mediaCapabilities−[object MediaCapabilities]", +// "getGamepads−function getGamepads() { [native code] }", +// "bluetooth−[object Bluetooth]", +// "share−function share() { [native code] }", +// "cookieEnabled−true", +// "virtualKeyboard−[object VirtualKeyboard]", +// "product−Gecko", +// "mediaDevices−[object MediaDevices]", +// "canShare−function canShare() { [native code] }", +// "getGamepads−function getGamepads() { [native code] }", +// "product−Gecko", +// "xr−[object XRSystem]", +// "clipboard−[object Clipboard]", +// "storageBuckets−[object StorageBucketManager]", +// "unregisterProtocolHandler−function unregisterProtocolHandler() { [native code] }", +// "productSub−20030107", +// "login−[object NavigatorLogin]", +// "vendorSub−", +// "login−[object NavigatorLogin]", +// "getInstalledRelatedApps−function getInstalledRelatedApps() { [native code] }", +// "mediaDevices−[object MediaDevices]", +// "locks−[object LockManager]", +// "webkitGetUserMedia−function webkitGetUserMedia() { [native code] }", +// "vendor−Google Inc.", +// "xr−[object XRSystem]", +// "mediaDevices−[object MediaDevices]", +// "virtualKeyboard−[object VirtualKeyboard]", +// "virtualKeyboard−[object VirtualKeyboard]", +// "appName−Netscape", +// "storageBuckets−[object StorageBucketManager]", +// "presentation−[object Presentation]", +// "onLine−true", +// "mimeTypes−[object MimeTypeArray]", +// "credentials−[object CredentialsContainer]", +// "presentation−[object Presentation]", +// "getGamepads−function getGamepads() { [native code] }", +// "vendorSub−", +// "virtualKeyboard−[object VirtualKeyboard]", +// "serviceWorker−[object ServiceWorkerContainer]", +// "xr−[object XRSystem]", +// "product−Gecko", +// "keyboard−[object Keyboard]", +// "gpu−[object GPU]", +// "getInstalledRelatedApps−function getInstalledRelatedApps() { [native code] }", +// "webkitPersistentStorage−[object DeprecatedStorageQuota]", +// "doNotTrack", +// "clearAppBadge−function clearAppBadge() { [native code] }", +// "presentation−[object Presentation]", +// "serial−[object Serial]", +// "locks−[object LockManager]", +// "requestMIDIAccess−function requestMIDIAccess() { [native code] }", +// "locks−[object LockManager]", +// "requestMediaKeySystemAccess−function requestMediaKeySystemAccess() { [native code] }", +// "vendor−Google Inc.", +// "pdfViewerEnabled−true", +// "language−zh-CN", +// "setAppBadge−function setAppBadge() { [native code] }", +// "geolocation−[object Geolocation]", +// "userAgentData−[object NavigatorUAData]", +// "mediaCapabilities−[object MediaCapabilities]", +// "requestMIDIAccess−function requestMIDIAccess() { [native code] }", +// "getUserMedia−function getUserMedia() { [native code] }", +// "mediaDevices−[object MediaDevices]", +// "webkitPersistentStorage−[object DeprecatedStorageQuota]", +// "sendBeacon−function sendBeacon() { [native code] }", +// "hardwareConcurrency−32", +// "credentials−[object CredentialsContainer]", +// "storage−[object StorageManager]", +// "cookieEnabled−true", +// "pdfViewerEnabled−true", +// "windowControlsOverlay−[object WindowControlsOverlay]", +// "scheduling−[object Scheduling]", +// "pdfViewerEnabled−true", +// "hardwareConcurrency−32", +// "xr−[object XRSystem]", +// "webdriver−false", +// "getInstalledRelatedApps−function getInstalledRelatedApps() { [native code] }", +// "getInstalledRelatedApps−function getInstalledRelatedApps() { [native code] }", +// "bluetooth−[object Bluetooth]" +// ] + +// document_key = ['_reactListeningo743lnnpvdg', 'location'] + +// window_key = [ +// "0", +// "window", +// "self", +// "document", +// "name", +// "location", +// "customElements", +// "history", +// "navigation", +// "locationbar", +// "menubar", +// "personalbar", +// "scrollbars", +// "statusbar", +// "toolbar", +// "status", +// "closed", +// "frames", +// "length", +// "top", +// "opener", +// "parent", +// "frameElement", +// "navigator", +// "origin", +// "external", +// "screen", +// "innerWidth", +// "innerHeight", +// "scrollX", +// "pageXOffset", +// "scrollY", +// "pageYOffset", +// "visualViewport", +// "screenX", +// "screenY", +// "outerWidth", +// "outerHeight", +// "devicePixelRatio", +// "clientInformation", +// "screenLeft", +// "screenTop", +// "styleMedia", +// "onsearch", +// "isSecureContext", +// "trustedTypes", +// "performance", +// "onappinstalled", +// "onbeforeinstallprompt", +// "crypto", +// "indexedDB", +// "sessionStorage", +// "localStorage", +// "onbeforexrselect", +// "onabort", +// "onbeforeinput", +// "onbeforematch", +// "onbeforetoggle", +// "onblur", +// "oncancel", +// "oncanplay", +// "oncanplaythrough", +// "onchange", +// "onclick", +// "onclose", +// "oncontentvisibilityautostatechange", +// "oncontextlost", +// "oncontextmenu", +// "oncontextrestored", +// "oncuechange", +// "ondblclick", +// "ondrag", +// "ondragend", +// "ondragenter", +// "ondragleave", +// "ondragover", +// "ondragstart", +// "ondrop", +// "ondurationchange", +// "onemptied", +// "onended", +// "onerror", +// "onfocus", +// "onformdata", +// "oninput", +// "oninvalid", +// "onkeydown", +// "onkeypress", +// "onkeyup", +// "onload", +// "onloadeddata", +// "onloadedmetadata", +// "onloadstart", +// "onmousedown", +// "onmouseenter", +// "onmouseleave", +// "onmousemove", +// "onmouseout", +// "onmouseover", +// "onmouseup", +// "onmousewheel", +// "onpause", +// "onplay", +// "onplaying", +// "onprogress", +// "onratechange", +// "onreset", +// "onresize", +// "onscroll", +// "onsecuritypolicyviolation", +// "onseeked", +// "onseeking", +// "onselect", +// "onslotchange", +// "onstalled", +// "onsubmit", +// "onsuspend", +// "ontimeupdate", +// "ontoggle", +// "onvolumechange", +// "onwaiting", +// "onwebkitanimationend", +// "onwebkitanimationiteration", +// "onwebkitanimationstart", +// "onwebkittransitionend", +// "onwheel", +// "onauxclick", +// "ongotpointercapture", +// "onlostpointercapture", +// "onpointerdown", +// "onpointermove", +// "onpointerrawupdate", +// "onpointerup", +// "onpointercancel", +// "onpointerover", +// "onpointerout", +// "onpointerenter", +// "onpointerleave", +// "onselectstart", +// "onselectionchange", +// "onanimationend", +// "onanimationiteration", +// "onanimationstart", +// "ontransitionrun", +// "ontransitionstart", +// "ontransitionend", +// "ontransitioncancel", +// "onafterprint", +// "onbeforeprint", +// "onbeforeunload", +// "onhashchange", +// "onlanguagechange", +// "onmessage", +// "onmessageerror", +// "onoffline", +// "ononline", +// "onpagehide", +// "onpageshow", +// "onpopstate", +// "onrejectionhandled", +// "onstorage", +// "onunhandledrejection", +// "onunload", +// "crossOriginIsolated", +// "scheduler", +// "alert", +// "atob", +// "blur", +// "btoa", +// "cancelAnimationFrame", +// "cancelIdleCallback", +// "captureEvents", +// "clearInterval", +// "clearTimeout", +// "close", +// "confirm", +// "createImageBitmap", +// "fetch", +// "find", +// "focus", +// "getComputedStyle", +// "getSelection", +// "matchMedia", +// "moveBy", +// "moveTo", +// "open", +// "postMessage", +// "print", +// "prompt", +// "queueMicrotask", +// "releaseEvents", +// "reportError", +// "requestAnimationFrame", +// "requestIdleCallback", +// "resizeBy", +// "resizeTo", +// "scroll", +// "scrollBy", +// "scrollTo", +// "setInterval", +// "setTimeout", +// "stop", +// "structuredClone", +// "webkitCancelAnimationFrame", +// "webkitRequestAnimationFrame", +// "chrome", +// "caches", +// "cookieStore", +// "ondevicemotion", +// "ondeviceorientation", +// "ondeviceorientationabsolute", +// "launchQueue", +// "documentPictureInPicture", +// "getScreenDetails", +// "queryLocalFonts", +// "showDirectoryPicker", +// "showOpenFilePicker", +// "showSaveFilePicker", +// "originAgentCluster", +// "onpageswap", +// "onpagereveal", +// "credentialless", +// "speechSynthesis", +// "onscrollend", +// "webkitRequestFileSystem", +// "webkitResolveLocalFileSystemURL", +// "sendMsgToSolverCS", +// "webpackChunk_N_E", +// "__next_set_public_path__", +// "next", +// "__NEXT_DATA__", +// "__SSG_MANIFEST_CB", +// "__NEXT_P", +// "_N_E", +// "regeneratorRuntime", +// "__REACT_INTL_CONTEXT__", +// "DD_RUM", +// "_", +// "filterCSS", +// "filterXSS", +// "__SEGMENT_INSPECTOR__", +// "__NEXT_PRELOADREADY", +// "Intercom", +// "__MIDDLEWARE_MATCHERS", +// "__STATSIG_SDK__", +// "__STATSIG_JS_SDK__", +// "__STATSIG_RERENDER_OVERRIDE__", +// "_oaiHandleSessionExpired", +// "__BUILD_MANIFEST", +// "__SSG_MANIFEST", +// "__intercomAssignLocation", +// "__intercomReloadLocation" +// ] + +use rand::Rng; + +const CORES: &[usize] = &[8, 16, 24, 32]; + +const CACHED_SCRIPTS: &[&str] = &["https://cdn.oaistatic.com/_next/static/cXh69klOLzS0Gy2joLDRS/_ssgManifest.js?dpl=453ebaec0d44c2decab71692e1bfe39be35a24b3"]; + +const CACHED_DPL: &[&str] = &["prod-f501fe933b3edf57aea882da888e1a544df99840"]; + +const NAVIGATOR_KEY: &[&str] = &[ + "registerProtocolHandler−function registerProtocolHandler() { [native code] }", + "storage−[object StorageManager]", + "locks−[object LockManager]", + "appCodeName−Mozilla", + "permissions−[object Permissions]", + "share−function share() { [native code] }", + "webdriver−false", + "managed−[object NavigatorManagedData]", + "canShare−function canShare() { [native code] }", + "vendor−Google Inc.", + "vendor−Google Inc.", + "mediaDevices−[object MediaDevices]", + "vibrate−function vibrate() { [native code] }", + "storageBuckets−[object StorageBucketManager]", + "mediaCapabilities−[object MediaCapabilities]", + "getGamepads−function getGamepads() { [native code] }", + "bluetooth−[object Bluetooth]", + "share−function share() { [native code] }", + "cookieEnabled−true", + "pdfViewerEnabled−true", + "windowControlsOverlay−[object WindowControlsOverlay]", + "scheduling−[object Scheduling]", + "pdfViewerEnabled−true", + "hardwareConcurrency−32", + "xr−[object XRSystem]", + "webdriver−false", + "getInstalledRelatedApps−function getInstalledRelatedApps() { [native code] }", + "getInstalledRelatedApps−function getInstalledRelatedApps() { [native code] }", + "bluetooth−[object Bluetooth]" +]; + +const DOCUMENT_KEY: &[&str] = &["_reactListeningo743lnnpvdg", "location"]; + +const WINDOW_KEY: &[&str] = &[ + "0", + "window", + "self", + "document", + "name", + "location", + "customElements", + "history", + "navigation", + "locationbar", + "menubar", + "personalbar", + "scrollbars", + "statusbar", + "toolbar", + "status", + "closed", + "frames", + "length", + "top", + "opener", + "parent", + "frameElement", + "navigator", + "origin", + "external", + "screen", + "innerWidth", + "innerHeight", + "scrollX", + "pageXOffset", + "scrollY", + "pageYOffset", + "visualViewport", + "screenX", + "screenY", + "outerWidth", + "outerHeight", + "devicePixelRatio", + "clientInformation", + "screenLeft", + "screenTop", + "styleMedia", + "onsearch", + "isSecureContext", + "trustedTypes", + "performance", + "onappinstalled", + "onbeforeinstallprompt", + "crypto", + "indexedDB", + "sessionStorage", + "localStorage", + "onbeforexrselect", + "onabort", + "onbeforeinput", + "onbeforematch", + "onbeforetoggle", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontentvisibilityautostatechange", + "oncontextlost", + "oncontextmenu", + "oncontextrestored", + "oncuechange", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onformdata", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadstart", + "onmousedown", + "onmouseenter", + "onmouseleave", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onpause", + "onplay", + "onplaying", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onscroll", + "onsecuritypolicyviolation", + "onseeked", + "onseeking", + "onselect", + "onslotchange", + "onstalled", + "onsubmit", + "onsuspend", + "ontimeupdate", + "ontoggle", + "onvolumechange", + "onwaiting", + "onwebkitanimationend", + "onwebkitanimationiteration", + "onwebkitanimationstart", + "onwebkittransitionend", + "onwheel", + "onauxclick", + "ongotpointercapture", + "onlostpointercapture", + "onpointerdown", + "onpointermove", + "onpointerrawupdate", + "onpointerup", + "onpointercancel", + "onpointerover", + "onpointerout", + "onpointerenter", + "onpointerleave", + "onselectstart", + "onselectionchange", + "onanimationend", + "onanimationiteration", + "onanimationstart", + "ontransitionrun", + "ontransitionstart", + "ontransitionend", + "ontransitioncancel", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onhashchange", + "onlanguagechange", + "onmessage", + "onmessageerror", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpopstate", + "onrejectionhandled", + "onstorage", + "onunhandledrejection", + "onunload", + "crossOriginIsolated", + "scheduler", + "alert", + "atob", + "blur", + "btoa", + "cancelAnimationFrame", + "cancelIdleCallback", + "captureEvents", + "clearInterval", + "clearTimeout", + "close", + "confirm", + "createImageBitmap", + "fetch", + "find", + "focus", + "getComputedStyle", + "getSelection", + "matchMedia", + "moveBy", + "moveTo", + "open", + "postMessage", + "print", + "prompt", + "queueMicrotask", + "releaseEvents", + "reportError", + "requestAnimationFrame", + "requestIdleCallback", + "resizeBy", + "resizeTo", + "scroll", + "scrollBy", + "scrollTo", + "setInterval", + "setTimeout", + "stop", + "structuredClone", + "webkitCancelAnimationFrame", + "webkitRequestAnimationFrame", + "chrome", + "caches", + "cookieStore", + "ondevicemotion", + "ondeviceorientation", + "ondeviceorientationabsolute", + "launchQueue", + "documentPictureInPicture", + "getScreenDetails", + "queryLocalFonts", + "showDirectoryPicker", + "showOpenFilePicker", + "showSaveFilePicker", + "originAgentCluster", + "onpageswap", + "onpagereveal", + "credentialless", + "speechSynthesis", + "onscrollend", + "webkitRequestFileSystem", + "webkitResolveLocalFileSystemURL", + "sendMsgToSolverCS", + "webpackChunk_N_E", + "__next_set_public_path__", + "next", + "__NEXT_DATA__", + "__SSG_MANIFEST_CB", + "__NEXT_P", + "_N_E", + "regeneratorRuntime", + "__REACT_INTL_CONTEXT__", + "DD_RUM", + "_", + "filterCSS", + "filterXSS", + "__SEGMENT_INSPECTOR__", + "__NEXT_PRELOADREADY", + "Intercom", + "__MIDDLEWARE_MATCHERS", + "__STATSIG_SDK__", + "__STATSIG_JS_SDK__", + "__STATSIG_RERENDER_OVERRIDE__", + "_oaiHandleSessionExpired", + "__BUILD_MANIFEST", + "__SSG_MANIFEST", + "__intercomAssignLocation", + "__intercomReloadLocation" +]; + +pub fn get_random_core_count() -> usize { + return CORES[rand::rng().random_range(0..CORES.len())]; +} + + +pub fn get_random_navigator_key() -> String { + return NAVIGATOR_KEY[rand::rng().random_range(0..NAVIGATOR_KEY.len())].to_string(); +} + +pub fn get_random_document_key() -> String { + return DOCUMENT_KEY[rand::rng().random_range(0..DOCUMENT_KEY.len())].to_string(); +} + +pub fn get_random_window_key() -> String { + return WINDOW_KEY[rand::rng().random_range(0..WINDOW_KEY.len())].to_string(); +} + +pub fn get_random_cached_script() -> String { + return CACHED_SCRIPTS[rand::rng().random_range(0..CACHED_SCRIPTS.len())].to_string(); +} diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/mod.rs b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/mod.rs new file mode 100644 index 0000000000..0bf2def16b --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/mod.rs @@ -0,0 +1,3 @@ +pub mod request; +pub mod token; +pub(crate) mod config; diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs new file mode 100644 index 0000000000..2af50353e3 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs @@ -0,0 +1,226 @@ +use crate::sentinel_refresh::generate::config::{get_random_core_count, get_random_document_key, get_random_navigator_key, get_random_window_key}; +use chrono::FixedOffset; +use chrono::Utc; +use idempotency::uuid::generate_random_uuid; +use rand::Rng; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use sha3::{Digest, Sha3_512}; + +// [ +// 4000, // first +// "Fri Apr 11 2025 13:28:14 GMT+0530 (India Standard Time)", // second +// None, // third +// 28, // fourth +// "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:137.0) Gecko/20100101 Firefox/137.0", // fifth +// "https://sora-cdn.oaistatic.com/_next/static/chunks/f90f2e60-61da6d33257661c4.js", // sixth +// None, // seventh +// "en-US", // eighth +// "en-US,en", // ninth +// 29, // tenth +// "productSub−20100101", // eleventh +// "location", // twelfth +// "ondrop", // thirteenth +// 10520, // fourteenth - generated from time.perf_counter() * 1000 +// "48094014-a024-4d2d-8ec4-ba9c7cfef9c4", // fifteenth +// "", // sixteenth +// 16, // seventeenth - Number of cores +#[derive(Debug, Clone)] +pub struct GenerateSentinelRefreshRequest { + first: usize, + second: String, // Date + third: Option, + fourth: Option, + fifth: String, // User agent + sixth: String, // URL + seventh: Option, // Language + eighth: String, // Accept-Language + ninth: String, // Product Sub + tenth: Option, + eleventh: String, // Product Sub - Navigator key + twelfth: String, // Location - Document key + thirteenth: String, // On Drop - Window key + fourteenth: usize, // generated from time.perf_counter() * 1000 + fifteenth: String, // UUID + sixteenth: String, // empty string + seventeenth: usize, // Number of cores +} + +fn generate_date() -> String { + let now_utc = Utc::now(); + let today_utc = now_utc.date_naive(); + let offset = FixedOffset::west_opt(5 * 60 * 60).unwrap(); + let now_with_offset = Utc::now().with_timezone(&offset); + + let formatted_date = now_with_offset.format("%a %b %d %Y %H:%M:%S GMT-0500 (Eastern Standard Time)"); + return formatted_date.to_string(); +} + +impl GenerateSentinelRefreshRequest { + pub fn new() -> Self { + let first_candidates = vec![1920 + 1080, 2560 + 1440, 1920 + 1200, 2560 + 1600]; + let first = first_candidates[rand::rng().random_range(0..first_candidates.len())]; + + let second = generate_date(); + + let third = None; + + let fourth = None; + + let fifth = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:137.0) Gecko/20100101 Firefox/137.0".to_string(); + + let sixth = "https://sora-cdn.oaistatic.com/_next/static/chunks/f90f2e60-61da6d33257661c4.js".to_string(); + + let seventh = None; + + let eighth = "en-US".to_string(); + + let ninth = "en-US,en".to_string(); + + let tenth = None; + + let eleventh = get_random_navigator_key(); + + let twelfth = get_random_document_key(); + + let thirteenth = get_random_window_key(); + + let fourteenth = 10520; + + let fifteenth = generate_random_uuid(); + + let sixteenth = "".to_string(); + + let seventeenth = get_random_core_count(); + Self { first, second, third, fourth, fifth, sixth, seventh, eighth, ninth, tenth, eleventh: eleventh.to_string(), twelfth: twelfth.to_string(), thirteenth: thirteenth.to_string(), fourteenth, fifteenth, sixteenth, seventeenth } + } + + pub fn with_fourth(mut self, fourth: usize) -> Self { + self.fourth = Some(fourth); + self + } + + pub fn with_tenth(mut self, tenth: usize) -> Self { + self.tenth = Some(tenth); + self + } + + //TODO(kasisnu): This is a temporary function to build the request. + // It is extremely hacky and this whole struct should be rewritten to be more ergonomic. + pub fn build(self) -> String { + let mut request = String::new(); + request.push_str("["); + request.push_str(&self.first.to_string()); + request.push_str(","); + request.push_str(&string_with_quotes(self.second)); + request.push_str(","); + request.push_str(&self.third.unwrap_or("null".to_string())); + request.push_str(","); + request.push_str(&self.fourth.unwrap_or(0).to_string()); + request.push_str(","); + request.push_str(&string_with_quotes(self.fifth)); + request.push_str(","); + request.push_str(&string_with_quotes(self.sixth)); + request.push_str(","); + request.push_str(&self.seventh.unwrap_or("null".to_string())); + request.push_str(","); + request.push_str(&string_with_quotes(self.eighth)); + request.push_str(","); + request.push_str(&string_with_quotes(self.ninth)); + request.push_str(","); + request.push_str(&self.tenth.unwrap_or(0).to_string()); + request.push_str(","); + request.push_str(&string_with_quotes(self.eleventh)); + request.push_str(","); + request.push_str(&string_with_quotes(self.twelfth)); + request.push_str(","); + request.push_str(&string_with_quotes(self.thirteenth)); + request.push_str(","); + request.push_str(&self.fourteenth.to_string()); + request.push_str(","); + request.push_str(&string_with_quotes(self.fifteenth)); + request.push_str(","); + request.push_str(&string_with_quotes(self.sixteenth)); + request.push_str(","); + request.push_str(&self.seventeenth.to_string()); + request.push_str("]"); + request + } + + pub fn hash_request(seed: f64, request: String) -> Vec { + let mut hasher = Sha3_512::new(); + hasher.update(seed.to_string()); + let base64_request = BASE64_STANDARD.encode(request.as_bytes()); + hasher.update(base64_request); + hasher.finalize().to_vec() + } + + pub fn with_fourth_and_tenth(mut self) -> (Self, String) { + let mut fourth = 0; + let mut tenth = 0; + let max_iterations = 500000; + let diff_len = 6; + let target_bytes = vec![0x0f, 0xff, 0xff]; + let random_seed: f64 = rand::rng().random(); + + let mut base64_request = BASE64_STANDARD.encode(self.clone().build().as_bytes()); + for i in 0..max_iterations { + fourth = i; + tenth = i >> 1; + let request = self.clone().with_fourth(fourth).with_tenth(tenth).build(); + let hash_bytes = Self::hash_request(random_seed, request.clone()); + base64_request = BASE64_STANDARD.encode(request.as_bytes()); + + let mut is_less_or_equal = true; + for j in 0..std::cmp::min(diff_len, target_bytes.len()) { + if j < hash_bytes.len() { + if hash_bytes[j] > target_bytes[j] { + is_less_or_equal = false; + break; + } else if hash_bytes[j] < target_bytes[j] { + break; + } + } + } + + if is_less_or_equal { + break; + } + } + + (self.with_fourth(fourth).with_tenth(tenth), format!("gAAAAAC{}", base64_request)) + } +} + +fn string_with_quotes(value: String) -> String { + format!("\"{}\"", value) +} +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_sentinel_refresh_request() { + let request = GenerateSentinelRefreshRequest::new(); + println!("{}", request.build()); + } + + #[test] + fn test_with_fourth_and_tenth() { + let request = GenerateSentinelRefreshRequest::new().with_fourth(100).with_tenth(200); + println!("{}", request.build()); + } + + #[test] + fn test_hash_request() { + let request = GenerateSentinelRefreshRequest::new().with_fourth(100).with_tenth(200); + let hash = GenerateSentinelRefreshRequest::hash_request(0.1, request.build()); + println!("{}", hash.iter().map(|b| format!("{:02x}", b)).collect::()); + } + + #[test] + fn test_with_fourth_and_tenth_2() { + let (request, base64_request) = GenerateSentinelRefreshRequest::new().with_fourth_and_tenth(); + println!("{}", base64_request); + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/token.rs b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/token.rs new file mode 100644 index 0000000000..6d39c9dc32 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/token.rs @@ -0,0 +1,68 @@ +use errors::AnyhowResult; +use serde_derive::{Deserialize, Serialize}; +use thiserror::Error; +use idempotency::uuid::generate_random_uuid; + +const SORA_IMAGE_GEN_URL: &str = "https://chatgpt.com/backend-api/sentinel/req"; + +/// This user agent is tied to the sentinel generation. If we need to change it, we may need to change sentinel generation too. +const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"; + +const SENTINEL_FLOW: &str = "sora_create_task"; + + +#[derive(Debug, Serialize, Deserialize)] +pub struct SentinelRequest { + #[serde(rename = "p")] + problem: String, + + #[serde(rename = "id")] + id: String, + + #[serde(rename = "flow")] + flow: String, +} + + +impl SentinelRequest { + pub fn new(problem: String) -> Self { + let id = generate_random_uuid(); + Self { problem, id, flow: SENTINEL_FLOW.to_string() } + } +} + + +#[derive(Debug, Serialize, Deserialize)] +pub struct SentinelResponse { + persona: String, + token: String, +} + + +pub async fn generate_token(problem: String) -> AnyhowResult { + let request = SentinelRequest::new(problem); + let client = reqwest::Client::new(); + let response = client.post(SORA_IMAGE_GEN_URL) + .header("User-Agent", USER_AGENT) + .header("Content-Type", "application/json") + .json(&request) + .send() + .await?; + + let response_json: SentinelResponse = response.json().await?; + Ok(response_json.token) +} + + +#[cfg(test)] +mod tests { + use super::*; + use crate::sentinel_refresh::generate::request::GenerateSentinelRefreshRequest; + + #[tokio::test] + async fn test_generate_token() { + let (_request, base64_request) = GenerateSentinelRefreshRequest::new().with_fourth_and_tenth(); + let token = generate_token(base64_request.to_string()).await.unwrap(); + println!("{}", token); + } +} \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs index 9e0c618fda..869169304e 100644 --- a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs +++ b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs @@ -1,2 +1,4 @@ pub use refresh::refresh_sentinel; -pub mod refresh; \ No newline at end of file +pub mod refresh; + +pub mod generate; \ No newline at end of file