Merge remote-tracking branch 'services/design/premium-website' into design/idea-hero-spiral

-top-[5px] centers the crossbar on the section's top rule (was 1px
high), and z-[31] lifts the marks over the frosted eyebrow band
(z-30), which was swallowing the right tick's downward stem.

Claude-Session: https://claude.ai/code/session_01YBoaBEt3gynzTwogBFtn4B

.github/workflows/artcraft-macos-publish.yml

@@ -1,240 +0,0 @@
-name: "Artcraft MacOS Publish"
-
-on:
-  push:
-    branches:
-      - release
-      - artcraft-release
-      - artcraft-macos  # Additional branch for MacOS testing
-
-# This workflow creates Universal macOS builds with signing certificate support.
-# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
-
-jobs:
-  publish-artcraft:
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Universal macOS builds (arm64 and x86_64)
-          - platform: 'macos-latest-xlarge'
-            args: '--target universal-apple-darwin'
-          # Uncomment below for additional platforms
-          # - platform: 'ubuntu-22.04'
-          #   args: ''
-          # - platform: 'windows-latest'
-          #   args: ''
-
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-
-      - name: install Rust stable
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          targets: ${{ matrix.platform == 'macos-latest-xlarge' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
-
-      - name: install dependencies (ubuntu only)
-        if: matrix.platform == 'ubuntu-22.04'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
-
-      - name: replace tauri config with mac-specific version
-        run: |
-          pushd ${{ github.workspace }}/crates/desktop/artcraft
-          # Use the mac-specific tauri conf for the build
-          # TODO(bt): Replace this with the platform-specific naming conventions
-          cp tauri-mac.conf.json tauri.conf.json
-          popd
-
-      - name: install frontend dependencies
-        run: |
-          # Navigate to the frontend directory to install dependencies
-          cd ./frontend
-
-          npm cache clean --force
-
-          echo "Installing all workspace dependencies..."
-          npm ci
-
-          echo "=== Version Information ==="
-          echo "Node version:"
-          node --version
-          echo "NPM version:"
-          npm --version
-
-          # Remove mismatched @nx packages and reinstall with correct versions
-          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
-
-          echo "=== Nx versions after adjustment ==="
-          npm list --depth=0 | grep nx
-
-      - name: import Apple Developer Certificate
-        if: matrix.platform == 'macos-latest-xlarge'
-        env:
-          APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
-          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
-          KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }}
-        run: |
-          echo "Starting certificate import process..."
-
-          # Create certificate file
-          echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
-
-          # Debug: Check if certificate file exists and has content
-          echo "Certificate file info:"
-          ls -la certificate.p12
-          file certificate.p12
-
-          # Debug: Check certificate contents without password (just structure)
-          echo "Certificate structure (may show error if password protected):"
-          openssl pkcs12 -info -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -noout || echo "Could not read certificate structure"
-
-          # Debug: Try to list what's in the p12 file
-          echo "Attempting to list certificate contents:"
-          security cms -D -i certificate.p12 || echo "Not a CMS file, trying PKCS12..."
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -info -nodes 2>&1 | head -20 || echo "Failed to read PKCS12 contents"
-
-          # Debug: Extract certificate details to verify type and expiration
-          echo "Extracting certificate details..."
-          # Extract just the certificate portion and examine it
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(Subject:|Issuer:|Not Before|Not After)" || echo "Failed to parse certificate details"
-
-          # Also try to see what type of certificate this is
-          echo "Certificate type check:"
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nokeys 2>/dev/null | openssl x509 -text -noout | grep -E "(CN=|OU=)" || echo "Could not determine certificate type"
-
-          # Debug: Check if private key exists
-          echo "Checking for private key..."
-          openssl pkcs12 -in certificate.p12 -passin pass:$APPLE_CERTIFICATE_PASSWORD -nocerts -nodes 2>&1 | grep -E "(PRIVATE KEY|RSA KEY)" | head -5 || echo "No private key found"
-
-          # Create temporary keychain
-          security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-          security default-keychain -s build.keychain
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-          security set-keychain-settings -t 3600 -u build.keychain
-
-          # Import certificate - ensure both certificate and private key are imported
-          echo "Importing certificate into keychain..."
-          # Import the p12 directly - this is the most reliable method
-          security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/security -T /usr/bin/productbuild
-
-          # Alternative: Try with additional flags if the first import didn't create an identity
-          echo "Verifying import..."
-          if ! security find-identity -v build.keychain | grep -q "1 valid identities found"; then
-            echo "First import didn't create identity, trying with -A flag..."
-            security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -A
-          fi
-
-          # Set trust settings for the certificate
-          echo "Setting trust settings..."
-          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
-
-          # Unlock the keychain again to ensure it's accessible
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
-
-          # Add the keychain to the search list
-          security list-keychains -d user -s build.keychain $(security list-keychains -d user | sed 's/^[[:space:]]*"//;s/"$//')
-
-          # List all certificates to debug
-          echo "Listing all keychains:"
-          security list-keychains
-
-          echo "Listing all certificates in build keychain:"
-          security find-certificate -a -p build.keychain || echo "No certificates found"
-
-          echo "Detailed certificate info:"
-          security find-certificate -c "Learning Machines" build.keychain || echo "Certificate 'Learning Machines' not found"
-
-          # Check certificate trust settings
-          echo "Checking certificate trust settings:"
-          security dump-trust-settings || echo "No custom trust settings"
-
-          # Try to find codesigning identities
-          echo "Finding all identities in keychain:"
-          security find-identity -v build.keychain || echo "No identities found"
-
-          echo "Finding codesigning identities:"
-          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
-
-          # Additional debug: Check if this is a valid Developer ID cert
-          echo "Checking for Developer ID certificates:"
-          security find-certificate -a build.keychain | grep -A 5 -B 5 "Developer ID" || echo "No Developer ID certificates found"
-
-      - name: verify certificate
-        if: matrix.platform == 'macos-latest-xlarge'
-        run: |
-          echo "=== Certificate Verification ==="
-
-          echo "1. Checking codesigning identities:"
-          security find-identity -v -p codesigning build.keychain || echo "No codesigning identities found"
-
-          echo -e "\n2. Checking all identities:"
-          security find-identity -v build.keychain || echo "No identities found"
-
-          echo -e "\n3. Checking certificates in keychain:"
-          security find-certificate -c "Learning Machines" build.keychain -p | openssl x509 -text -noout | grep -E "(Subject:|CN=)" || echo "Could not parse certificate"
-
-          # Set the correct signing identity
-          echo "APPLE_SIGNING_IDENTITY=Developer ID Application: Learning Machines LLC (DJ6XS33FX8)" >> $GITHUB_ENV
-          echo "Proceeding with identity: Developer ID Application: Learning Machines LLC (DJ6XS33FX8)"
-
-          # Also set the team ID
-          echo "APPLE_DEVELOPMENT_TEAM=DJ6XS33FX8" >> $GITHUB_ENV
-
-      - name: Pre-build frontend to avoid Tauri build issues
-        run: |
-          # Build the frontend app from the correct directory
-          cd ./frontend
-
-          # Increase Node.js memory limit for the build
-          export NODE_OPTIONS="--max-old-space-size=8192"
-
-          # Use npx to ensure we use the local nx version
-          echo "Building with npx nx..."
-          npx nx run artcraft:build
-
-          echo "Checking build output..."
-          ls -la apps/artcraft/dist/ | head -10
-
-          # Update Tauri config to skip the build (since we already built)
-          cd ${{ github.workspace }}/crates/desktop/artcraft
-          cp tauri.conf.json tauri.conf.json.bak
-          # Set beforeBuildCommand to empty or a no-op command
-          if command -v jq &> /dev/null; then
-            jq '.build.beforeBuildCommand = ""' tauri.conf.json > tauri.conf.json.tmp
-            mv tauri.conf.json.tmp tauri.conf.json
-          else
-            node -e "const fs = require('fs'); const config = JSON.parse(fs.readFileSync('tauri.conf.json', 'utf8')); config.build.beforeBuildCommand = ''; fs.writeFileSync('tauri.conf.json', JSON.stringify(config, null, 2));"
-          fi
-
-
-      - name: build and publish
-        uses: tauri-apps/tauri-action@v0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          APPLE_ID: ${{ secrets.APPLE_ID }}
-          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-          APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
-          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
-          APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
-          APPLE_SIGNING_IDENTITY: ${{ env.APPLE_SIGNING_IDENTITY }}
-          APPLE_DEVELOPMENT_TEAM: ${{ env.APPLE_DEVELOPMENT_TEAM }}
-          # Use the SQLx query cache for SQLite
-          SQLX_OFFLINE: true
-        with:
-          projectPath: ./crates/desktop/artcraft
-          tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
-          releaseName: "ArtCraft v__VERSION__"
-          releaseBody: "See the assets below to download this version and install."
-          releaseDraft: true
-          prerelease: false
-          args: ${{ matrix.args }}

.github/workflows/artcraft-windows-publish.yml

@@ -1,107 +0,0 @@
-name: "Artcraft Windows Publish"
-
-on:
-  push:
-    branches:
-      - release
-      - artcraft-release
-      - artcraft-windows  # Additional branch for Windows testing
-
-# This workflow creates Windows builds.
-# On each push to the release branches it will create or update a GitHub release, build your app, and upload the artifacts to the release.
-
-jobs:
-  publish-artcraft-windows:
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Windows builds
-          - platform: 'windows-latest'
-            args: ''
-
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: install Rust stable
-        uses: dtolnay/rust-toolchain@stable
-
-      - name: install wreq dependencies
-        run: |
-          choco install cmake -y
-          choco install strawberryperl -y
-          choco install pkgconfiglite -y
-          choco install llvm -y
-          choco install nasm -y
-        shell: cmd
-
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-
-      - name: install frontend dependencies
-        run: |
-          # Navigate to the frontend directory to install dependencies
-          cd ./frontend
-
-          npm cache clean --force
-
-          echo "Installing all workspace dependencies..."
-          npm ci
-
-          echo "=== Version Information ==="
-          echo "Node version:"
-          node --version
-          echo "NPM version:"
-          npm --version
-
-          # Remove mismatched @nx packages and reinstall with correct versions
-          npm install --save-dev @nx/remix@21.1.1 @nx/js@21.1.1 @nx/react@21.1.1 @nx/vite@21.1.1 @nx/eslint@21.1.1
-
-          echo "=== Nx versions after adjustment ==="
-          npm list --depth=0 | findstr nx
-
-      - name: Pre-build frontend to avoid Tauri build issues
-        run: |
-          # Build the frontend app from the correct directory
-          cd ./frontend
-
-          # Increase Node.js memory limit for the build
-          $env:NODE_OPTIONS="--max-old-space-size=8192"
-
-          # Use npx to ensure we use the local nx version
-          echo "Building with npx nx..."
-          npx nx run artcraft:build
-
-          echo "Checking build output..."
-          dir apps/artcraft/dist/ | Select-Object -First 10
-
-          # Update Tauri config to skip the build (since we already built)
-          cd ${{ github.workspace }}/crates/desktop/artcraft
-          Copy-Item tauri.conf.json tauri.conf.json.bak
-
-          # Set beforeBuildCommand to empty using PowerShell
-          $config = Get-Content tauri.conf.json | ConvertFrom-Json
-          $config.build.beforeBuildCommand = ""
-          $config | ConvertTo-Json -Depth 10 | Set-Content tauri.conf.json
-
-      - name: build and publish
-        uses: tauri-apps/tauri-action@v0
-        env:
-          # NB(bt): This is needed by wreq. Hopefully it doesn't kill Tauri
-          RUSTFLAGS: "-C target-feature=+crt-static"
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # Use the SQLx query cache for SQLite
-          SQLX_OFFLINE: true
-        with:
-          projectPath: ./crates/desktop/artcraft
-          tagName: artcraft-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
-          releaseName: "ArtCraft v__VERSION__"
-          releaseBody: "Production build of ArtCraft. See the assets below to download and install."
-          releaseDraft: true
-          prerelease: false
-          args: ${{ matrix.args }}

.github/workflows/docker-publish-cpu.yml

@@ -26,7 +26,7 @@ on:
       - 'includes/**'
       - 'test_data/**'
 env:
-  IMAGE_NAME: storyteller-rust
+  IMAGE_NAME: artcraft-services
 
 jobs:
   # Push image to GitHub Packages.
@@ -82,7 +82,7 @@ jobs:
         name: Compute Docker tags
         uses: docker/metadata-action@v5
         with:
-          images: ghcr.io/storytold/storyteller-rust
+          images: ghcr.io/storytold/artcraft-services
           tags: |
             type=raw,value=latest
             type=raw,value=${{ github.sha }}

.github/workflows_inactive/tauri-publish-2d.yml

@@ -1,63 +0,0 @@
-name: 'publish'
-
-on:
-  push:
-    branches:
-      - tauri-prod-release
-
-# This workflow will trigger on each push to the `release` branch to create or update a GitHub release, build your app, and upload the artifacts to the release.
-
-jobs:
-  publish-tauri:
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: 'macos-latest' # for Arm based macs (M1 and above).
-            args: '--target aarch64-apple-darwin'
-          # - platform: 'macos-latest' # for Intel based macs.
-          #   args: '--target x86_64-apple-darwin'
-          # - platform: 'ubuntu-22.04' # for Tauri v1 you could replace this with ubuntu-20.04.
-          #   args: ''
-          # - platform: 'windows-latest'
-          #   args: ''
-
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: lts/*
-
-      - name: install Rust stable
-        uses: dtolnay/rust-toolchain@stable
-        with:
-          # Those targets are only used on macos runners so it's in an `if` to slightly speed up windows and linux builds.
-          targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }}
-
-      - name: install dependencies (ubuntu only)
-        if: matrix.platform == 'ubuntu-22.04' # This must match the platform value defined above.
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libwebkit2gtk-4.0-dev libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
-        # webkitgtk 4.0 is for Tauri v1 - webkitgtk 4.1 is for Tauri v2.
-        # You can remove the one that doesn't apply to your app to speed up the workflow a bit.
-
-      - name: install frontend dependencies
-        run: cd ./crates/desktop/artcraft && yarn install # change this to npm, pnpm or bun depending on which one you use.
-
-      - uses: tauri-apps/tauri-action@v0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          projectPath: ./crates/desktop/artcraft
-          tagName: app-v__VERSION__ # the action automatically replaces \_\_VERSION\_\_ with the app version.
-          releaseName: 'App v__VERSION__'
-          releaseBody: 'See the assets to download this version and install.'
-          releaseDraft: true
-          prerelease: false
-          args: ${{ matrix.args }}

.gitignore

@@ -29,7 +29,6 @@ twitch_secrets.toml
 production_database_secret_connect.sh
 production_database_secret_query.sh
 production_database_voice_clone_report_download.sh
-script/artcraft/mac_signing_build.sh
 script/production_database_secret_backup.sh
 script/production_database_secret_connect.sh
 script/socat_dev_database_connect.sh
@@ -40,6 +39,7 @@ venv/
 # LLM Testing
 /external/
 /prompts/
+/untracked/
 /reports/
 /requests/
 
@@ -56,9 +56,6 @@ crates/lib/sqlite_queries/target/
 # Javascript build directories
 node_modules/
 
-# Frontend build outputs
-frontend/apps/artcraft/bundle-report.html
-
 # Vim noise
 *.swo
 *.swp

.sqlx/query-13d435d8f681b73e4080748dcef0579e246e41049af67e1bd79ffc06aa6ba974.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET is_dismissed_by_user = 1\n    WHERE id = ?\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 1
-    },
-    "nullable": []
-  },
-  "hash": "13d435d8f681b73e4080748dcef0579e246e41049af67e1bd79ffc06aa6ba974"
-}

.sqlx/query-1e3758851fc8451670f86a9432009b73bdb81de4a08dcaa96a86afb5f9be7a68.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET\n      task_status = ?,\n      on_complete_batch_token = ?,\n      on_complete_primary_media_file_token = ?,\n      on_complete_primary_media_file_class = ?,\n      on_complete_primary_media_file_cdn_url = ?,\n      on_complete_primary_media_file_thumbnail_url_template = ?\n    WHERE\n      provider = ?\n      AND provider_job_id = ?\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 8
-    },
-    "nullable": []
-  },
-  "hash": "1e3758851fc8451670f86a9432009b73bdb81de4a08dcaa96a86afb5f9be7a68"
-}

.sqlx/query-2436eb7bf4b0e23a899311a67a77f71d89eb35461e2113e43cd9d221bfa91c9a.json

@@ -1,128 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    SELECT\n      id,\n      task_status,\n      task_type,\n      model_type,\n      provider,\n      provider_job_id,\n      frontend_caller,\n      frontend_subscriber_id,\n      frontend_subscriber_payload,\n      on_complete_primary_media_file_token,\n      on_complete_primary_media_file_class,\n      on_complete_batch_token,\n      on_complete_primary_media_file_cdn_url,\n      on_complete_primary_media_file_thumbnail_url_template,\n      on_failure_type,\n      on_failure_message,\n      created_at as \"created_at: DateTime<Utc>\",\n      updated_at as \"updated_at: DateTime<Utc>\",\n      completed_at as \"completed_at: DateTime<Utc>\"\n    FROM tasks\n    WHERE is_dismissed_by_user == 0\n  ",
-  "describe": {
-    "columns": [
-      {
-        "name": "id",
-        "ordinal": 0,
-        "type_info": "Text"
-      },
-      {
-        "name": "task_status",
-        "ordinal": 1,
-        "type_info": "Text"
-      },
-      {
-        "name": "task_type",
-        "ordinal": 2,
-        "type_info": "Text"
-      },
-      {
-        "name": "model_type",
-        "ordinal": 3,
-        "type_info": "Text"
-      },
-      {
-        "name": "provider",
-        "ordinal": 4,
-        "type_info": "Text"
-      },
-      {
-        "name": "provider_job_id",
-        "ordinal": 5,
-        "type_info": "Text"
-      },
-      {
-        "name": "frontend_caller",
-        "ordinal": 6,
-        "type_info": "Text"
-      },
-      {
-        "name": "frontend_subscriber_id",
-        "ordinal": 7,
-        "type_info": "Text"
-      },
-      {
-        "name": "frontend_subscriber_payload",
-        "ordinal": 8,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_complete_primary_media_file_token",
-        "ordinal": 9,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_complete_primary_media_file_class",
-        "ordinal": 10,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_complete_batch_token",
-        "ordinal": 11,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_complete_primary_media_file_cdn_url",
-        "ordinal": 12,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_complete_primary_media_file_thumbnail_url_template",
-        "ordinal": 13,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_failure_type",
-        "ordinal": 14,
-        "type_info": "Text"
-      },
-      {
-        "name": "on_failure_message",
-        "ordinal": 15,
-        "type_info": "Text"
-      },
-      {
-        "name": "created_at: DateTime<Utc>",
-        "ordinal": 16,
-        "type_info": "Int64"
-      },
-      {
-        "name": "updated_at: DateTime<Utc>",
-        "ordinal": 17,
-        "type_info": "Int64"
-      },
-      {
-        "name": "completed_at: DateTime<Utc>",
-        "ordinal": 18,
-        "type_info": "Int64"
-      }
-    ],
-    "parameters": {
-      "Right": 0
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "2436eb7bf4b0e23a899311a67a77f71d89eb35461e2113e43cd9d221bfa91c9a"
-}

.sqlx/query-30272128d83543d9fbfca930b7cdd4af30adfb4d90f2de044463a111c6340495.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET task_status = \"cancelled_by_user\"\n    WHERE task_status NOT IN (\n        \"complete_success\",\n        \"complete_failure\",\n        \"dead\",\n        \"cancelled_by_user\",\n        \"cancelled_by_provider\",\n        \"cancelled_by_us\"\n    )\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 0
-    },
-    "nullable": []
-  },
-  "hash": "30272128d83543d9fbfca930b7cdd4af30adfb4d90f2de044463a111c6340495"
-}

.sqlx/query-35677894f7b02f32a1bf6396fa6d10caba69bb0f3f03ee7ffbfa1437689d42df.json

@@ -1,86 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    SELECT\n      id,\n      task_status,\n      task_type,\n      model_type,\n      provider,\n      provider_job_id,\n      frontend_caller,\n      queue_status_url,\n      queue_response_url,\n      prompt_token,\n      frontend_subscriber_id,\n      frontend_subscriber_payload\n    FROM tasks\n    WHERE\n      provider = ?\n      AND provider_job_id = ?\n  ",
-  "describe": {
-    "columns": [
-      {
-        "name": "id",
-        "ordinal": 0,
-        "type_info": "Text"
-      },
-      {
-        "name": "task_status",
-        "ordinal": 1,
-        "type_info": "Text"
-      },
-      {
-        "name": "task_type",
-        "ordinal": 2,
-        "type_info": "Text"
-      },
-      {
-        "name": "model_type",
-        "ordinal": 3,
-        "type_info": "Text"
-      },
-      {
-        "name": "provider",
-        "ordinal": 4,
-        "type_info": "Text"
-      },
-      {
-        "name": "provider_job_id",
-        "ordinal": 5,
-        "type_info": "Text"
-      },
-      {
-        "name": "frontend_caller",
-        "ordinal": 6,
-        "type_info": "Text"
-      },
-      {
-        "name": "queue_status_url",
-        "ordinal": 7,
-        "type_info": "Text"
-      },
-      {
-        "name": "queue_response_url",
-        "ordinal": 8,
-        "type_info": "Text"
-      },
-      {
-        "name": "prompt_token",
-        "ordinal": 9,
-        "type_info": "Text"
-      },
-      {
-        "name": "frontend_subscriber_id",
-        "ordinal": 10,
-        "type_info": "Text"
-      },
-      {
-        "name": "frontend_subscriber_payload",
-        "ordinal": 11,
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Right": 2
-    },
-    "nullable": [
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "35677894f7b02f32a1bf6396fa6d10caba69bb0f3f03ee7ffbfa1437689d42df"
-}

.sqlx/query-4fc9674ae39abaadc36054c200290ec95d0e2c7765336f0fa97805a2265f5c4c.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    INSERT INTO tasks (\n      id,\n      task_status,\n      task_type,\n      model_type,\n      provider,\n      provider_job_id,\n      queue_status_url,\n      queue_response_url,\n      prompt_token,\n      frontend_caller,\n      frontend_subscriber_id,\n      frontend_subscriber_payload\n    )\n    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 12
-    },
-    "nullable": []
-  },
-  "hash": "4fc9674ae39abaadc36054c200290ec95d0e2c7765336f0fa97805a2265f5c4c"
-}

.sqlx/query-50a2449b87abed7df3302a74ecd0e8ab87940077a247636680cb09b2fe488321.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET\n      task_status = ?,\n      on_complete_batch_token = ?,\n      on_complete_primary_media_file_token = ?,\n      on_complete_primary_media_file_class = ?,\n      on_complete_primary_media_file_cdn_url = ?,\n      on_complete_primary_media_file_thumbnail_url_template = ?\n    WHERE id = ?\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 7
-    },
-    "nullable": []
-  },
-  "hash": "50a2449b87abed7df3302a74ecd0e8ab87940077a247636680cb09b2fe488321"
-}

.sqlx/query-9b99ccd0baf9a4b74d8903f63e3a3dcf4c09b75cad978f0532388cbd743b7516.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET task_status = ?,\n    on_failure_type = ?,\n    on_failure_message = ?\n    WHERE id = ?\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 4
-    },
-    "nullable": []
-  },
-  "hash": "9b99ccd0baf9a4b74d8903f63e3a3dcf4c09b75cad978f0532388cbd743b7516"
-}

.sqlx/query-dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET task_status = ?\n    WHERE id = ?\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 2
-    },
-    "nullable": []
-  },
-  "hash": "dc0367e9e43c3d7dda3623e77337ab4529eac2192475be7d08589886703ba1cb"
-}

.sqlx/query-f83018ec5493982659a56576296914f937bb4e75db3a522eb4bda486d5caa402.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "\n    UPDATE tasks\n    SET is_dismissed_by_user = 1\n    WHERE is_dismissed_by_user != 1\n  ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Right": 0
-    },
-    "nullable": []
-  },
-  "hash": "f83018ec5493982659a56576296914f937bb4e75db3a522eb4bda486d5caa402"
-}

AGENTS.md

@@ -4,54 +4,38 @@ This file provides guidance for coding agents when working with the Artcraft mon
 
 ## Project Overview
 
-Artcraft is a web and desktop application for generating AI image and video. It is written in 
-Rust and TypeScript and contains desktop, server, and frontend components.
+ArtCraft Services contains the Rust backend, HTTP API, workers, and TypeScript web
+frontends for ArtCraft. The Tauri desktop application is maintained separately at
+https://github.com/storytold/artcraft.
 
 ## Project Structure
 
-```
-artcraft/
-├── _database/                           # Schema definitions and migrations (MySQL, SQLite, Elasticsearch, etc.)
-│   ├── elasticsearch/                   # Elasticsearch schema and queries
-│   └── sql/                             # MySQL and SQLite schema definitions and migrations
-│       ├── artcraft_migrations/         # ArtCraft desktop app SQLite migrations
-│       ├── migrations/                  # Server MySQL migrations
-│       └── migrations_squashed/         # Fully materialized MySQL schema definitions for most tables
-├── _tools/                              # Various 3rd party tool integrations and configurations
-│   └── postman/                         # Postman configs for test HTTP requests against development and production
-├── build/                               # Dockerfile build instructions for server components
-├── crates/                              # Rust workspace
-│   ├── api_clients/                     # HTTP clients for calling internal and 3rd party services
-│   ├── cli/                             # Command line tools
-│   ├── desktop/                         # Desktop (Tauri) apps
-│   │   └── artcraft/                    # (Important) ArtCraft, the desktop app. This is one of our main pieces of software
-│   ├── lib/                             # Various utility libraries for servers, CLI tools, desktop, etc.
-│   ├── schema/                          # Data definition layer: MySQL, SQlite, Redis, S3/R2 buckets, etc.
-│   │   ├── buckets/                     # Declares R2 cloud bucket topology
-│   │   ├── database/                    # MySQL, SQLite, Elasticsearch, Redis, etc.
-│   │   │   ├── elasticsearch_schema/    # Elasticsearch
-│   │   │   ├── migration/               # (deprecated) Online schema adapters for MySQL 
-│   │   │   ├── mysql_queries/           # Sqlx MySQL queries for our backend monolith `storyteller-web`, jobs, etc.
-│   │   │   ├── redis_common/            # Redis support
-│   │   │   ├── redis_schema/            # Redis key and HKEY topology
-│   │   │   ├── sqlite_queries/          # (deprecated) Sqlite queries
-│   │   │   └── sqlite_tasks/            # Queries for the ArtCraft desktop app's "tasks" database.
-│   │   ├── public/                      # Token identifier and enum variant definitions
-│   │   │   ├── composite_identifier/    # MySQL composite key system
-│   │   │   ├── enums/                   # MySQL "enums" stored in VARCHAR fields.
-│   │   │   └── tokens/                  # Primary database identifiers with Stripe-like ID prefixes, eg. "user_{entropy}"
-│   │   └── service/                     # Backend HTTP services and jobs
-│   │       ├── job/                     # Backend jobs.
-│   │       │   └── video_thumbnail_job/ # Render video thumbnails
-│   │       ├── plugins/                 # Collections of reusable Actix-Web HTTP functions (user and billing systems)
-│   │       └── web/                     # HTTP web servers
-│   │           └── storyteller_web/     # (Important) Our main HTTP API monolith and backend.
-│   └── frontend/                        # Nx typescript monorepo for our websites and Tauri desktop apps
-│       ├── apps/                        # Websites and Tauri desktop apps
-│       │   ├── artcraft/                # ArtCraft the Tauri app's frontend. Used in conjunction with `artcraft` the Rust crate.
-│       │   └── artcraft-website/        # The website for https://getartcraft.com
-│       └── libs/                        # Support libraries, reusable React components, etc.
-└── Cargo.toml                           # Rust monorepo workspace
+```text
+artcraft-services/
+├── _database/                    # SQL migrations and Elasticsearch schemas
+│   └── sql/
+│       ├── migrations/           # Server MySQL migrations
+│       └── migrations_squashed/  # Materialized MySQL table definitions
+├── _tools/postman/               # HTTP request collections
+├── build/                        # Service Dockerfiles
+├── crates/
+│   ├── api_clients/              # ArtCraft API types, clients, router, provider clients
+│   ├── cli/                      # Development and operations tools
+│   ├── lib/                      # Shared Rust utilities
+│   ├── schema/                   # Database queries, public tokens/enums, bucket paths
+│   │   └── database/
+│   │       ├── mysql_queries/    # SQLx queries shared by services, jobs, and CLI tools
+│   │       ├── redis_common/     # Redis support
+│   │       └── redis_schema/     # Redis key and HKEY topology
+│   └── service/
+│       ├── job/                  # Provider workers, media processing, analytics
+│       ├── plugins/              # Shared billing and service components
+│       └── web/storyteller_web/  # Main Actix Web HTTP API
+├── frontend/
+│   ├── apps/artcraft-webapp/     # Browser application at app.getartcraft.com
+│   ├── apps/artcraft-website/    # Product website at getartcraft.com
+│   └── libs/                     # Shared React components and TypeScript libraries
+└── Cargo.toml                    # Rust workspace
 ```
 
 ## Code Style

Cargo.toml

@@ -18,19 +18,15 @@ members = [
   "crates/api_clients/google_drive_downloader",
   "crates/api_clients/google_sign_in",
   "crates/api_clients/grok_api_client",
-  "crates/api_clients/grok_consumer_client",
   "crates/api_clients/gmicloud_client",
-  "crates/api_clients/midjourney_client",
   "crates/api_clients/muapi_client",
   "crates/api_clients/beeble_client",
-  "crates/api_clients/openai_sora_client",
   "crates/api_clients/openart_client",
   "crates/api_clients/rootly_client",
   "crates/api_clients/runwayml_client",
   "crates/api_clients/kinovi_web_client",
   "crates/api_clients/thumbnail_generator",
   "crates/api_clients/worldlabs_api_client",
-  "crates/api_clients/worldlabs_consumer_client",
 
   # CLI tools for operations, etc.
   "crates/cli/devenv/dev-database-seed",
@@ -44,27 +40,19 @@ members = [
   "crates/cli/production/migration-tool",
   "crates/cli/production/support-tool",
 
-  # Desktop apps
-  "crates/desktop/artcraft",
-
   # Shared libraries
   "crates/lib/actix_artcraft",
   "crates/lib/actix_cors_configs",
   "crates/lib/actix_helpers",
   "crates/lib/bootstrap",
-  "crates/lib/build_metadata",
   "crates/lib/caching/memory_caching",
-  "crates/lib/caching/memory_store",
   "crates/lib/caching/redis_caching",
   "crates/lib/cloudflare_errors",
-  "crates/lib/cloudflare_mitigation",
   "crates/lib/concurrency",
-  "crates/lib/cookie_store",
   "crates/lib/data/collections",
   "crates/lib/data/crockford",
   "crates/lib/data/datetimes",
   "crates/lib/data/primitives",
-  "crates/lib/data/web_base64",
   "crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve.
   "crates/lib/error_utils",
   "crates/lib/errors",
@@ -123,7 +111,6 @@ members = [
   "crates/schema/database/mysql_testing",
   "crates/schema/database/redis_common",
   "crates/schema/database/redis_schema",
-  "crates/schema/database/sqlite_tasks",
   #"crates/schema/database/sqlite_queries",
 
   # Schema (public): common strongly-typed tokens, ids, payloads, etc.
@@ -167,7 +154,6 @@ members = [
 
   # Vendored code
   "crates/vendor/limitation",
-  "crates/vendor/tauri-plugin-http",
   "crates/vendor/Symphonia/symphonia",
   "crates/vendor/Symphonia/symphonia-bundle-flac",
   "crates/vendor/Symphonia/symphonia-bundle-mp3",
@@ -245,14 +231,11 @@ artcraft_router = { path = "crates/api_clients/artcraft/artcraft_router" }
 bootstrap = { path = "crates/lib/bootstrap" }
 bucket_client = { path = "crates/api_clients/bucket_client" }
 bucket_paths = { path = "crates/schema/buckets/bucket_paths" }
-build_metadata = { path = "crates/lib/build_metadata" }
 cloud_storage = { path = "crates/api_clients/cloud_storage" }
 cloudflare_errors = { path = "crates/lib/cloudflare_errors" }
-cloudflare_mitigation = { path = "crates/lib/cloudflare_mitigation" }
 collections = { path = "crates/lib/data/collections" }
 component_traits = { path = "crates/service/plugins/component_traits" }
 concurrency = { path = "crates/lib/concurrency" }
-cookie_store = { path = "crates/lib/cookie_store" }
 crockford = { path = "crates/lib/data/crockford" }
 datadog_client = { path = "crates/api_clients/datadog_client" }
 easyenv = { path = "crates/lib/easyenv" }
@@ -266,7 +249,6 @@ file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" }
 filesys = { path = "crates/lib/files/filesys" }
 beeble_client = { path = "crates/api_clients/beeble_client" }
 grok_api_client = { path = "crates/api_clients/grok_api_client" }
-grok_consumer_client = { path = "crates/api_clients/grok_consumer_client" }
 rootly_client = { path = "crates/api_clients/rootly_client" }
 rootly_config = { path = "crates/schema/config/rootly_config" }
 shared_env_var_config = { path = "crates/schema/config/shared_env_var_config" }
@@ -277,21 +259,17 @@ jwt_light = { path = "crates/lib/jwt_light" }
 jwt_signer = { path = "crates/lib/jwt_signer" }
 opaque_cursors = { path = "crates/lib/opaque_cursors" }
 memory_caching = { path = "crates/lib/caching/memory_caching" }
-memory_store = { path = "crates/lib/caching/memory_store" }
-midjourney_client = { path = "crates/api_clients/midjourney_client" }
 mimetypes = { path = "crates/lib/files/mimetypes" }
 muapi_client = { path = "crates/api_clients/muapi_client" }
 metrics = { path = "crates/lib/metrics" }
 mysql_queries = { path = "crates/schema/database/mysql_queries" }
 mysql_testing = { path = "crates/schema/database/mysql_testing" }
 pager = { path = "crates/lib/pager" }
-openai_sora_client = { path = "crates/api_clients/openai_sora_client" }
 primitives = { path = "crates/lib/data/primitives" }
 kinovi_web_client = { path = "crates/api_clients/kinovi_web_client" }
 server_environment = { path = "crates/lib/server_environment" }
 sha1_hash = { path = "crates/lib/sha1_hash" }
 shared_service_components = { path = "crates/lib/shared_service_components" }
-sqlite_tasks = { path = "crates/schema/database/sqlite_tasks" }
 subprocess_common = { path = "crates/lib/deprecated/subprocess_common" }
 test_data = { path = "crates/testing/test_data" }
 test_utils = { path = "crates/testing/test_utils" }
@@ -301,9 +279,7 @@ url_utils = { path = "crates/lib/url_utils" }
 users = { path = "crates/lib/users" }
 uuid_utils = { path = "crates/lib/uuid_utils" }
 video_info = { path = "crates/lib/video_info" }
-web_base64 = { path = "crates/lib/data/web_base64" }
 worldlabs_api_client = { path = "crates/api_clients/worldlabs_api_client" }
-worldlabs_consumer_client = { path = "crates/api_clients/worldlabs_consumer_client" }
 
 # External libs
 # Centrally pinned versions so that we can uniformly upgrade/downgrade versions across the workspace
@@ -320,8 +296,6 @@ anyhow = "1.0.102"
 assertor = "0.0.3"
 base64 = "0.22.1"
 bcrypt = "0.15"
-bstr = "1.12.1"
-byteorder = "1.5.0"
 bytes = "1.11.1"
 chrono = { version = "0.4.44", features = ["serde"] }
 clap = { version = "=4.5.60", features = ["derive"] }
@@ -333,7 +307,6 @@ env_logger = "0.11.9"
 futures = { version = "0.3.32" }
 hex-literal = "1.1.0"
 hostname = "0.4.0"
-indexmap = "2.13.0"
 log = "0.4.29"
 md-5 = "0.10.6" # insecure but used for gravatar
 mockall = "0.12.1"
@@ -355,11 +328,6 @@ sha2 = "0.11.0-rc.5"
 sha3 = "0.10.8"
 speculoos = "0.11.0"
 strum = { version = "0.28.0", features = ["derive"] }
-tauri = { version = "2.11.2", features = ["unstable", "devtools", "protocol-asset"] } # NB: "devtools" prevents App Store distribution; "ubstable" is needed for window access
-tauri-plugin-dialog = "2.7.1"
-tauri-plugin-fs = "2.5.1"
-tauri-plugin-log = "2.8.0"
-tauri-plugin-upload = "2.4.0"
 tempdir = "0.3.7"
 tempfile = "3.27.0"
 thiserror = "2.0.18"

README.md

@@ -1,82 +1,211 @@
-<p align="center">
-  <video src="https://github.com/user-attachments/assets/b4e24c27-d87d-4fd1-8599-dc0d0b8af48d" width="100%" autoplay="true" loop controls>
-</p>
+# ArtCraft Services
 
-<p align="center">The IDE for artists.</p>
-<p align="center">
-  <a href="https://discord.gg/artcraft"><img alt="Discord" src="https://img.shields.io/discord/1359579021108842617?style=for-the-badge&label=discord&color=ffffff&logo=discord&logoColor=ffffff" /></a>
-  <a href="https://www.youtube.com/@OfficialArtCraftStudios"><img alt="YouTube" src="https://img.shields.io/youtube/channel/subscribers/UCdjY4VG0ntoGwFsKZO4sVWA?style=for-the-badge&logo=YouTube" /></a>
-  <a href="https://x.com/intent/follow?screen_name=get_artcraft"><img alt="X" src="https://img.shields.io/twitter/follow/get_artcraft?style=for-the-badge&label=follow&logo=x&logoColor=ffffff&color=ffffff" /></a>
-  <a href="https://www.linkedin.com/company/artcraft-ai"><img alt="LinkedIn" src="https://img.shields.io/badge/linkedin--0A66C2?style=for-the-badge"></a>
-</p>
+This repository contains ArtCraft's backend, HTTP API, background workers, and web
+frontends. It is a Rust and TypeScript monorepo with shared API definitions,
+provider clients, database queries, and development tooling.
 
----
+The Tauri desktop application is maintained in
+[storytold/artcraft](https://github.com/storytold/artcraft), along with the product
+overview, feature demos, and desktop downloads.
 
-ArtCraft
-========
-ArtCraft is the IDE for interactive AI image and video creation.
-We turn prompting into *crafting*, so your ideas become a form of tangible expression and computing.
-This is Adobe Photoshop for everyone, and we're giving away the source code!
+## Backend architecture
 
-## Show, Don't Tell: Advanced Crafting Features
+[`storyteller-web`](./crates/service/web/storyteller_web) is the main HTTP service,
+built with Rust, Actix Web, and Tokio. It handles authentication, accounts,
+media uploads and libraries, generation requests, job status, credits, and Stripe
+billing. The service retains the `storyteller-web` name, and its hosted API uses
+`https://api.storyteller.ai`.
 
-Text-to-image is great, but artists *need control*. It's important to know what your image will look like before you generate it, and it's vitally important to achieve consistency and repeatability.
+Generation spans the HTTP service, provider integrations, and asynchronous workers:
 
-| Feature                           | Demo + Explanation                                                                                                                                                                                                                                                      
-|-----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Image to Location**             | ![Video](https://github.com/user-attachments/assets/21f103e3-cc19-4882-a630-9caa1b76ae31) Placing virtual actors into physical environments establishes single-location consistency. You can film multiple shots within a room without having things disappear.         |
-| **3D Image Compositing**          | ![Video](https://github.com/user-attachments/assets/f93a616f-571d-474e-bcc0-53736de7303d) Use images (backdrops, foreground elements, props, etc.) in scenes with depth and blend them naturally together. Just a couple of images usually leads to great compositions. |
-| **2D Image Compositing**          | ![Video](https://github.com/user-attachments/assets/d6f99391-e496-4c62-9e37-29734ba5f899) Use images, background removal, layers, and simple drawing tools to precisely compose a scene.                                                                                |
-| **Image to 3D Mesh**              | ![Video](https://github.com/user-attachments/assets/600a405c-e360-48c1-9b42-6e657ae6243b) It's almost impossible to lay out complicated objects or block complicated scenes; turning images into 3D helps position elements exactingly and intentionally.               |
-| **Character Posing**              | ![Video](https://raw.githubusercontent.com/storytold/github-media/22a27373707b13f48fc56405f1f89ca143c43d5d/character-posing.webp) You can dynamically pose your characters to achieve the precise character, scene, and camera blocking before calling "action".                                                |
-| **Scene Blocking w/ Kit Bashing** | ![Video](https://github.com/user-attachments/assets/eef025ac-0346-4a46-a023-d48e23629eb5) Use 3D asset kits to precisely block out your scene: get the correct angles, object positions, and rich depth layering you can't with text prompting.                         |
-| **Character Identity Transfer**   | ![Video](https://github.com/user-attachments/assets/629119ee-8c76-4a83-9827-8c6c995a3ec1) Use mannequins as simple 3D ControlNets for posing any character.                                                                                                             |
-| **Background Removal**            | ![Video](https://github.com/user-attachments/assets/90c65057-5531-404f-83af-b34e66e24ec1) Remove backgrounds from images to make them useful in 2D or 3D compositing. They can be props, layers, or backdrops.                                                          |
-| **Mixed Asset Crafting**          | ![Video](https://raw.githubusercontent.com/storytold/github-media/main/ship-editing.gif) You can use image cutouts, worlds, and simple 3D meshes all together to precisely and intentionally lay out your scenes.                                                       |
-| **Scene Blocking**                | (preview coming soon)                                                                                                                                                                                                                                                   |
-| **Canvas Editing**                | (preview coming soon)                                                                                                                                                                                                                                                   |
-| **Scene Relighting**              | (preview coming soon)                                                                                                                                                                                                                                                   |
+```mermaid
+flowchart LR
+  Clients[Web, desktop, and API clients] --> API[storyteller-web]
+  API --> Router[artcraft_router]
+  Router --> Providers[Generation providers]
+  Providers -->|Webhooks| API
+  Workers[Background workers] -->|Poll jobs| Providers
+  API --> DB[(MySQL)]
+  Workers --> DB
+  API --> Storage[(Object storage)]
+  Workers --> Storage
+```
 
-Note: all of the above videos were generated for free with Grok Video; the cost to build this README was negligible.
+A typical generation request follows this path:
 
-## Quick and Easy Prompting
-We haven't abandoned text-to-asset generation for quick prototyping and ideation. We support every popular workflow in a first class fashion.
+1. The API authenticates the caller, validates the request and input media, and
+   checks the user's access and credits.
+2. The generation pipeline calculates the cost, bills the wallet, and uses
+   [`artcraft_router`](./crates/api_clients/artcraft/artcraft_router) to build and
+   submit a provider-specific request. The API records inference jobs in MySQL
+   and returns job tokens to the client.
+3. Completion is handled through provider webhooks or polling workers, depending
+   on the integration. Results are downloaded into object storage, registered as
+   media files, and associated with the completed jobs. Separate workers handle
+   follow-up processing such as video thumbnails.
+4. Clients poll job-status endpoints and load the resulting media through CDN URLs.
 
-| Feature               | Demo + Explanation                                                                                                                                    
-|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Text to Image**     | ![Video](https://github.com/user-attachments/assets/9cc289cd-faf4-4eaf-aed2-21134cce127c) Text prompt over a dozen different image models.            |
-| **Image Editing**     | ![Video](https://github.com/user-attachments/assets/a06fa6ad-936c-42d0-8767-48fdbb8ff141) Edit with Nano Banana Pro and GPT Image 1.5.                |
-| **Image Editing**     | ![Video](https://github.com/user-attachments/assets/f036e08a-f3a6-417a-98ee-ec7f04b2b5ff) Use inpainting, drawing, masking, etc. to edit images.      |
-| **Image to Video**    | ![Video](https://github.com/user-attachments/assets/2bc6c592-511e-4fba-b40f-03c96699b7f7) Image to video with lots of different options and controls. |
-| **Image Inpainting**  | (preview coming soon)                                                                                                                                 |
-| **Image Ingredients** | (preview coming soon)                                                                                                                                 |
+Storage responsibilities are split across these components:
 
-## Models and Providers Supported within Artcraft
+| Component                  | Role                                                         |
+|----------------------------|--------------------------------------------------------------|
+| MySQL + SQLx               | Accounts, media metadata, inference jobs, wallets, and bills |
+| Redis                      | Caching, rate limiting, and job progress                     |
+| Elasticsearch              | Search indexes and queries                                   |
+| S3-compatible storage / R2 | Uploaded media, generated assets, and derived files          |
 
-| Provider   | Features                                                                                                                                                                |
-|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Artcraft   | Nano Banana, Nano Banana Pro, GPT-Image-1 / 1.5, Seedance 1.0 Lite / 1.5 Pro / 2.0, Seedream 4 / 4.5, Flux 1.1 / Kontext, Veo 2 / 3 / 3.1, Kling 1.6 / 2.1 / 2.5 / 2.6, Sora 2 / Pro, Hunyuan 3d 2 / 3 |
-| Grok       | Grok Imagine, Grok Video                                                                                                                                                |
-| Midjourney | Image Gen (all versions)                                                                                                                                                |
-| Sora       | Sora 1, Sora 2, GPT-Image-1                                                                                                                                             |
-| WorldLabs  | Marble (Gaussian Splat World Generation)                                                                                                                                |
+HTTP routes and handlers live in
+[`storyteller_web/src/http_server`](./crates/service/web/storyteller_web/src/http_server).
+MySQL queries belong in the shared
+[`mysql_queries`](./crates/schema/database/mysql_queries) crate so that handlers,
+workers, and CLI tools use the same data access layer. Reusable billing components
+live under [`crates/service/plugins`](./crates/service/plugins), and background
+services live under [`crates/service/job`](./crates/service/job).
 
-We're going to be adding the following providers soon: Kling (via Kling website accounts), Google (via API keys), 
-Runway (via website account), Luma (via website account).
+## HTTP API
 
-We're potentially interested in adding other aggregators for those who already have subscriptions and credits at 
-those providers, for example: OpenArt, FreePik, etc.
+The API exposes two generation interfaces:
 
-## Downloads
+- **Application endpoints:** `/v1/omni_gen/generate/*` uses user sessions for image,
+  video, audio, mesh, and splat generation. `/v1/omni_gen/models/*` and
+  `/v1/omni_gen/cost/*` expose model discovery and cost estimation without requiring
+  a user session. Application job status is available under `/v1/jobs`.
+- **Programmatic endpoints:** `/v1/omni_api` uses an API key in the `Authorization`
+  header. It provides image and video generation, image/video/audio uploads, and
+  job-status polling. Use `Authorization: Bearer <api-key>`; API access must be
+  enabled for the account.
 
-- [Visit our website for the stable Windows and MacOS releases](https://getartcraft.com/)
-- Or you can grab a [more recent Windows and MacOS build directly](https://github.com/storytold/artcraft/releases)
-- Linux requires building from source for now
+For example, a programmatic video request goes to
+`POST /v1/omni_api/generate/video`. The response contains an
+`inference_job_token`, which the caller polls with
+`GET /v1/omni_api/job_status/job/{token}`. The
+[Omni API guide](./_docs/omni_api/artcraft_omni_api.md) covers authentication,
+request and response bodies, URL inputs, and runnable examples.
 
-## Documentation
+Start with these sources when adding or tracing an endpoint:
 
-- [developer documentation](./_docs)
-- [tools, scripts, misc](./script)
-- [license](./LICENSE.md)
-- [roadmap](./ROADMAP.md)
+- [Route registration](./crates/service/web/storyteller_web/src/http_server/routes/application_routes)
+  maps URLs and HTTP methods to handlers for generation, media, users, billing,
+  API keys, and other service areas.
+- [API definitions](./crates/api_clients/artcraft/artcraft_api_defs) contain shared
+  Rust request, response, and error types.
+- [Rust API client](./crates/api_clients/artcraft/artcraft_client) and
+  [TypeScript API library](./frontend/libs/api) provide client implementations.
+- [Provider clients](./crates/api_clients) implement upstream HTTP integrations;
+  [the generation router](./crates/api_clients/artcraft/artcraft_router) adapts
+  generation requests and cost estimates across providers.
 
+## Frontend
+
+[`frontend`](./frontend) contains the Nx workspace for React and TypeScript apps
+and shared libraries. The main web frontends use Vite, with Zustand
+and signals for state, Three.js for 3D scenes, and shared UI and generation tools.
+
+| Path                             | Purpose                                       |
+|----------------------------------|-----------------------------------------------|
+| `frontend/apps/artcraft-webapp`  | Browser application at `app.getartcraft.com`  |
+| `frontend/apps/artcraft-website` | Product website at `getartcraft.com`          |
+| `frontend/libs/api`              | HTTP clients, API host selection, and models  |
+| `frontend/libs/omni-gen`         | Shared generation logic                       |
+| `frontend/libs/components`       | Reusable UI, editors, and generation controls |
+| `frontend/libs/tauri-api`        | Frontend bindings for native desktop commands |
+
+The web apps call the backend through the shared API library, which handles JSON
+and multipart requests and session credentials. Libraries such as `tauri-api` and
+`tauri-utils` remain because shared web components still import their types,
+helpers, and browser-compatible behavior. The native desktop app and libraries
+used only by that app live in the separate desktop repository.
+
+## Repository layout
+
+```text
+artcraft-services/
+├── crates/
+│   ├── service/web/       # HTTP services, including storyteller_web
+│   ├── service/job/       # Provider workers, media processing, analytics
+│   ├── service/plugins/   # Shared billing and service components
+│   ├── api_clients/       # ArtCraft API types, clients, router, provider clients
+│   ├── schema/            # Database access, public tokens/enums, bucket paths
+│   ├── lib/               # Shared Rust utilities
+│   └── cli/               # Development and operations tools
+├── frontend/
+│   ├── apps/              # Web frontends
+│   └── libs/              # Shared TypeScript libraries
+├── _database/             # SQL migrations, materialized schemas, search schemas
+├── _docs/                 # Setup guides and technical documentation
+├── _tools/postman/        # HTTP request collections
+├── build/                 # Service Dockerfiles
+├── script/                # Development, build, and database tooling
+└── Cargo.toml             # Rust workspace
+```
+
+## Local development
+
+Use Rust/Cargo for backend work and Node.js/npm for the main frontend workspace.
+See the [development setup guide](./_docs/dev_setup.md) for toolchain setup and
+[frontend README](./frontend/README.md) for dependency installation and Nx usage.
+
+### Backend
+
+The server needs a migrated MySQL database, Redis, Elasticsearch configuration,
+object storage, and credentials for the integrations being exercised. The
+[server setup guide](./_docs/dev_setup_server.md) covers local MySQL and Redis;
+the remaining configuration is defined in the
+[server config directory](./crates/service/web/storyteller_web/config) and
+[startup code](./crates/service/web/storyteller_web/src/startup).
+
+In development, the server loads `storyteller-web.common.env`,
+`storyteller-web.development.env`, and `storyteller-web.development-secrets.env`
+from its configuration search paths: the repository root, `./config`, and the
+server's config directory. Its bootstrap skips the root `.env` file.
+
+With the toolchain and service configuration in place, run from the repository root:
+
+```bash
+SQLX_OFFLINE=true cargo check -p storyteller-web
+SQLX_OFFLINE=true cargo run -p storyteller-web
+```
+
+The default bind address is `0.0.0.0:12345`, configurable through `BIND_ADDRESS`.
+`GET /_status` exposes the service health check. Provider polling and thumbnail
+processing require their corresponding worker processes and configuration.
+
+`SQLX_OFFLINE=true` uses the checked-in `.sqlx` query metadata during compilation;
+the running server still needs its databases. When changing SQLx queries, use
+[`script/rust/sqlx_codegen_database.sh`](./script/rust/sqlx_codegen_database.sh)
+to regenerate metadata against migrated development databases.
+
+### Web frontend
+
+To run the browser app against a local backend:
+
+```bash
+cd frontend
+npm install
+VITE_USE_LOCAL_API=true npx nx dev artcraft-webapp
+```
+
+The browser app runs at `http://localhost:4201`. `VITE_USE_LOCAL_API=true` selects
+`http://localhost:12345`; without that override, its Vite development proxy targets
+the hosted API. API host selection lives in
+[`StorytellerApiHostStore`](./frontend/libs/api/src/lib/config/StorytellerApiHostStore.ts).
+
+From `frontend`, build the web app or run the product website with:
+
+```bash
+npx nx build artcraft-webapp
+npx nx dev artcraft-website
+```
+
+The website runs at `http://localhost:4200`. Repository-root launchers are in
+[`script/website`](./script/website). For desktop development, use the
+[ArtCraft desktop repository](https://github.com/storytold/artcraft).
+
+## Further reading
+
+- [Technical documentation](./_docs)
+- [Code conventions](./_docs/code_conventions.md)
+- [Database schemas and migrations](./_database)
+- [Postman collections](./_tools/postman)
+- [Build and development scripts](./script)
+- [License](./LICENSE.md)

_database/sql/artcraft_migrations/20250708000000_create_tasks_table.sql

@@ -1,127 +0,0 @@
--- noinspection SqlDialectInspectionForFile
---
--- NB: This version needs to be bumped even if just adding or changing comments!
---  Failure to do so will cause Windows to segfault at start. (This is horrible!)
---
--- NB: `app_state_dir.rs` contains the file version number.
---
--- Migration history:
---   tasks_v1.sqlite - initial version
---   tasks_v2.sqlite - added model_type (nullable)
---   tasks_v3.sqlite - added frontend_caller, comments
---   tasks_v4.sqlite - added is_dismissed_by_user
---   tasks_v5.sqlite - added five fields:
---      on_complete_batch_token,
---      on_complete_primary_media_file_token,
---      on_complete_primary_media_file_class,
---      on_complete_primary_media_file_cdn_url,
---      on_complete_primary_media_file_thumbnail_url_template,
---   tasks_v6.sqlite - added two fields:
---      on_failure_type
---      on_failure_message
---   tasks_v7.sqlite - added prompt token + two fields (for fal):
---      prompt_token
---      queue_status_url
---      queue_response_url
-
-CREATE TABLE tasks (
-    -- Task auto-incrementing primary key.
-    -- We don't use this.
-    id TEXT NOT NULL PRIMARY KEY,
-
-    -- TaskStatus enum
-    -- This is how the job system manages job states.
-    -- e.g. pending, started, complete_success, dead, etc.
-    task_status TEXT NOT NULL,
-
-    -- TaskType enum
-    -- e.g. image_generation, video_generation, etc.
-    task_type TEXT NOT NULL,
-
-    -- TaskModelType enum
-    -- e.g. flux_1_dev, veo_2, etc.
-    model_type TEXT,
-
-    -- GenerationProvider enum
-    -- Together with `provider_job_id`, this creates a foreign key lookup to the job.
-    -- e.g. 'artcraft', 'fal', 'sora', etc.
-    provider TEXT,
-
-    -- The primary key for the job in the provider's system.
-    provider_job_id TEXT,
-
-    -- OPTIONAL.
-    -- For third party jobs, we need to record the prompt details upfront.
-    -- We create a prompt record and store the token to associate once the task completes.
-    prompt_token TEXT,
-
-    -- OPTIONAL.
-    -- Tell the job system which caller initiated the task.
-    -- e.g. 'canvas'
-    frontend_caller TEXT,
-
-    -- OPTIONAL.
-    -- An arbitrary opaque identifier that frontend can set.
-    frontend_subscriber_id TEXT,
-
-    -- OPTIONAL.
-    -- An opaque JSON payload set by the frontend subscriber that
-    -- will be re-emitted back to the frontend.
-    frontend_subscriber_payload TEXT,
-
-    -- Whether the user has dismissed the task from view.
-    is_dismissed_by_user INTEGER NOT NULL DEFAULT 0,
-
-    -- OPTIONAL.
-    -- For some third party systems (eg. FAL), the URL to check the status of the
-    -- job in the queue.
-    queue_status_url TEXT,
-
-    -- OPTIONAL.
-    -- For some third party systems (eg. FAL), the URL to check the
-    -- results (when the job completes)
-    queue_response_url TEXT,
-
-    -- OPTIONAL.
-    -- When the generation completes, the batch token (if any).
-    -- This will point to a collection of multiple files.
-    on_complete_batch_token TEXT,
-
-    -- OPTIONAL.
-    -- When the generation completes, the media token of the "primary" media file.
-    -- We'll probably always generate homogenous types, but just in case this is the first or primary item in a batch.
-    on_complete_primary_media_file_token TEXT,
-
-    -- OPTIONAL.
-    -- When the generation completes, the file type of the primary media file.
-    -- We'll probably always generate homogenous types, but just in case this is the first or primary item in a batch.
-    on_complete_primary_media_file_class TEXT,
-
-    -- OPTIONAL.
-    -- The URL to the media file.
-    on_complete_primary_media_file_cdn_url TEXT,
-
-    -- OPTIONAL.
-    -- The URL template for the thumbnail of the primary media file.
-    on_complete_primary_media_file_thumbnail_url_template TEXT,
-
-    -- OPTIONAL.
-    -- An enum-like string describing the failure reason (even for non-terminal failures).
-    -- The enum is `TaskFailureType`, which is a close analogue of the backend API's `FrontendFailureCategory`.
-    on_failure_type TEXT,
-
-    -- OPTIONAL.
-    -- A human-readable message describing the failure reason (even for non-terminal failures).
-    on_failure_message TEXT,
-
-    created_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
-    updated_at INTEGER NOT NULL DEFAULT (unixepoch('now')),
-    completed_at INTEGER DEFAULT NULL
-);
-
--- Unique Indices
-CREATE UNIQUE INDEX idx_tasks_on_provider_job_id ON tasks(provider_job_id);
-
--- Indices
-CREATE INDEX idx_tasks_on_status ON tasks(task_status);
-CREATE INDEX idx_tasks_on_provider ON tasks(provider);

_docs/dev_setup.md

@@ -1,45 +1,40 @@
-Dev Setup
-=========
+# Development setup
 
-# ArtCraft 
+This repository contains the ArtCraft backend and web frontends. Desktop development
+lives in [storytold/artcraft](https://github.com/storytold/artcraft).
 
-ArtCraft is a Rust / Tauri app.
+## Backend
 
-To set up the ArtCraft development environment,  install the following:
+Install Rust and Cargo, then follow the [server setup guide](./dev_setup_server.md)
+for MySQL and Redis. The server also needs Elasticsearch configuration, object
+storage, and credentials for the integrations you use. Configuration files live in
+[`storyteller_web/config`](../crates/service/web/storyteller_web/config); dependency
+initialization lives in [`startup`](../crates/service/web/storyteller_web/src/startup).
 
-1. [Install Rust](https://doc.rust-lang.org/cargo/getting-started/installation.html).
-2. [Install npm](https://nodejs.org/en/download) or [nvm](https://github.com/nvm-sh/nvm). (Node version `v24.13.0` works at time of writing.) 
-3. [Install nx](https://nx.dev/docs/getting-started/installation). (Nx version `v22.4.5` works at time of writing.)
-4. [Install Tauri CLI](https://v2.tauri.app/reference/cli/). (Version `tauri-cli 2.10.0` works at time of writing.)
-
-An easy way to get started with running the app in development is to run the two commands (in separate terminals):
-
-**Mac and Linux Development** 
+From the repository root, with those dependencies configured:
 
 ```bash
-# Run the frontend dev server
-./script/artcraft/unix_frontend_dev.sh
-
-# Run the Tauri Rust application
-./script/artcraft/unix_rust_dev.sh
+SQLX_OFFLINE=true cargo check -p storyteller-web
+SQLX_OFFLINE=true cargo run -p storyteller-web
 ```
 
-**Windows Development**
+The API listens on `http://localhost:12345` by default. `SQLX_OFFLINE=true` uses
+checked-in query metadata at build time; the server still needs MySQL at runtime.
+See the [root README](../README.md) for configuration loading and worker setup.
 
-```powershell
-# Run the frontend dev server
-.\script\artcraft\windows_frontend_dev.ps1
+## Web frontends
 
-# Run the Tauri Rust application
-.\script\artcraft\windows_rust_dev.ps1
+Install Node.js and npm, then run from the repository root:
+
+```bash
+cd frontend
+npm install
+VITE_USE_LOCAL_API=true npx nx dev artcraft-webapp
 ```
 
+The web app runs at `http://localhost:4201`, using the local backend. To develop
+against the hosted API, omit `VITE_USE_LOCAL_API=true`.
 
-# ArtCraft Server
-
-ArtCraft's server is a Rust / Actix app called `storyteller-web`.
-
-You don't need to run this to develop the ArtCraft application, but it can be useful to spin up 
-a development instance for adding new server functions or as your own private local copy.
-
-See [dev_setup_server.md](./dev_setup_server.md) for instructions.
+For the product website, run `npx nx dev artcraft-website` from `frontend`; it uses
+`http://localhost:4200`. Repository-root launchers live under [`script/website`](../script/website).
+See the [frontend README](../frontend/README.md) for builds and troubleshooting.

_docs/dev_setup_server.md

@@ -2,11 +2,9 @@
 
 ArtCraft's server is a Rust / Actix app called `storyteller-web`.
 
-You don't need to run this to develop the ArtCraft application, but it can be useful to spin up
-a development instance for adding new server functions or as your own private local copy.
-
-`storyteller-web` can function entirely within the development environment, and ArtCraft can be
-pointed to a local instance of the server.
+Run it locally to develop backend functionality or connect the web app to your
+own API instance. The desktop app is maintained in
+[storytold/artcraft](https://github.com/storytold/artcraft).
 
 **TODO**: Concisely describe setting up server components. Note: we have
 previous docs in `old/` that may still be relevant, though they're perhaps
@@ -53,7 +51,7 @@ Then verify access with `./script/dev_mysql_connect.sh`
 ```bash
 cargo install diesel_cli \
   --no-default-features \
-  --features mysql,sqlite
+  --features mysql
 ```
 
 If this fails,
@@ -90,7 +88,7 @@ Finally, you'll need the sqlx CLI tool to run codegen.
 You likely won't need this now, but if you change any queries, this will be necessary:
 
 ```bash
-cargo install sqlx-cli --features rustls,mysql,sqlite
+cargo install sqlx-cli --version 0.7.4 --no-default-features --features rustls,mysql --locked
 ```
 
 ### Install Redis (Mac)

crates/AGENTS.md

@@ -50,7 +50,6 @@ helpers go in the parent `mod tests` so sub-modules can `use super::*`.
 ## Key Crates
 
 - `storyteller_web` — main HTTP API monolith (actix-web)
-- `artcraft` (desktop) — Tauri desktop app
 - `artcraft_router` — provider routing for image/video generation (Artcraft, Fal, KinoviWeb, Muapi)
 - `artcraft_api_defs` — shared HTTP API type definitions
 - `kinovi_web_client` — HTTP client for the Kinovi/KinoviWeb video generation service

crates/api_clients/artcraft/artcraft_router/src/generate/generate_video/providers/kinovi/seedance_2p5/build.rs

@@ -1,9 +1,11 @@
 use kinovi_web_client::generate::video::generate_seedance_2p5::{
   KinoviSeedance2p5AspectRatio as KinoviAspectRatio,
+  KinoviSeedance2p5Bitrate,
   KinoviSeedance2p5OutputResolution as KinoviOutputResolution,
 };
 
 use crate::api::router_aspect_ratio::RouterAspectRatio;
+use crate::api::router_bitrate::RouterBitrate;
 use crate::api::router_resolution::RouterResolution;
 use crate::client::request_mismatch_mitigation_strategy::RequestMismatchMitigationStrategy;
 use crate::errors::artcraft_router_error::ArtcraftRouterError;
@@ -64,8 +66,10 @@ fn do_build_kinovi_seedance_2p5(mut builder: GenerateVideoRequestBuilder) -> Res
   let total_input_seconds = builder.total_reference_video_input_seconds
     .map(|seconds| u8::try_from(seconds).unwrap_or(u8::MAX));
 
-  // NB: `builder.bitrate` is intentionally ignored — 2.5 has no bitrate
-  // option (and bitrate never affects cost).
+  let maybe_bitrate = match builder.bitrate.take() {
+    Some(RouterBitrate::High) => Some(KinoviSeedance2p5Bitrate::High),
+    Some(RouterBitrate::Normal) | None => None,
+  };
 
   let unhandled_request_state = KinoviSeedance2p5RemainingItems {
     start_frame: builder.start_frame.take(),
@@ -81,6 +85,7 @@ fn do_build_kinovi_seedance_2p5(mut builder: GenerateVideoRequestBuilder) -> Res
     duration_seconds,
     prompt,
     total_input_seconds,
+    maybe_bitrate,
     unhandled_request_state: Some(unhandled_request_state),
   })
 }
@@ -246,6 +251,7 @@ fn plan_duration(
 
 #[cfg(test)]
 mod tests {
+  use kinovi_web_client::creds::kinovi_web_session::KinoviWebSession;
   use kinovi_web_client::generate::video::generate_seedance_2p5::{
     KinoviSeedance2p5AspectRatio as KinoviAspectRatio,
     KinoviSeedance2p5OutputResolution as KinoviOutputResolution,
@@ -261,13 +267,50 @@ mod tests {
   use crate::api::router_video_model::RouterVideoModel;
   use crate::api::video_list_ref::VideoListRef;
   use crate::client::request_mismatch_mitigation_strategy::RequestMismatchMitigationStrategy;
+  use crate::client::router_client::RouterClient;
+  use crate::client::router_kinovi_web_client::RouterKinoviWebClient;
   use crate::generate::generate_video::generate_video_request_builder::GenerateVideoRequestBuilder;
+  use crate::generate::generate_video::providers::kinovi::seedance_2p5::cost::KinoviSeedance2p5CostState;
   use crate::generate::generate_video::providers::kinovi::seedance_2p5::draft::KinoviSeedance2p5DraftState;
   use crate::generate::generate_video::video_generation_draft::VideoGenerationDraftRequest;
+  use crate::generate::generate_video::video_generation_draft_context::VideoGenerationDraftContext;
   use crate::generate::generate_video::video_generation_draft_or_request::VideoGenerationDraftOrRequest;
 
   use super::*;
 
+  #[tokio::test]
+  async fn bitrate_survives_finalization_without_changing_cost() {
+    let client = RouterClient::KinoviWeb(RouterKinoviWebClient::new(
+      KinoviWebSession::from_cookies_string(String::new()),
+    ));
+    let context = VideoGenerationDraftContext {
+      client: Some(&client),
+      ..Default::default()
+    };
+    let baseline = unwrap_draft(build_kinovi_seedance_2p5(base_builder()));
+    let baseline_cost = KinoviSeedance2p5CostState::from_draft(&baseline).estimate_cost();
+
+    for bitrate in [None, Some(RouterBitrate::Normal), Some(RouterBitrate::High)] {
+      let mut draft = unwrap_draft(build_kinovi_seedance_2p5(GenerateVideoRequestBuilder {
+        bitrate,
+        ..base_builder()
+      }));
+      let draft_cost = KinoviSeedance2p5CostState::from_draft(&draft).estimate_cost();
+      // No media inputs: finalization needs no network calls or credentials.
+      let finalized = draft.to_request(&context).await.expect("finalize should succeed");
+
+      assert_eq!(
+        matches!(finalized.request.maybe_bitrate, Some(KinoviSeedance2p5Bitrate::High)),
+        matches!(bitrate, Some(RouterBitrate::High)),
+      );
+      let final_cost = KinoviSeedance2p5CostState::from_request(&finalized).estimate_cost();
+      for cost in [draft_cost, final_cost] {
+        assert_eq!(cost.cost_in_credits, baseline_cost.cost_in_credits);
+        assert_eq!(cost.cost_in_usd_cents, baseline_cost.cost_in_usd_cents);
+      }
+    }
+  }
+
   mod materialized_field_conversions {
     use super::*;
 

crates/api_clients/artcraft/artcraft_router/src/generate/generate_video/providers/kinovi/seedance_2p5/cost.rs

@@ -91,6 +91,7 @@ impl KinoviSeedance2p5CostState {
       duration_seconds: self.duration_seconds,
       total_input_seconds,
       use_face_blur_hack: None,
+      maybe_bitrate: None,
     };
 
     // Enterprise tier: what generations actually cost us (our discounted

crates/api_clients/artcraft/artcraft_router/src/generate/generate_video/providers/kinovi/seedance_2p5/draft.rs

@@ -1,5 +1,5 @@
 use kinovi_web_client::generate::video::generate_seedance_2p5::{
-  GenerateSeedance2p5Request, KinoviSeedance2p5AspectRatio, KinoviSeedance2p5Modality,
+  GenerateSeedance2p5Request, KinoviSeedance2p5AspectRatio, KinoviSeedance2p5Bitrate, KinoviSeedance2p5Modality,
   KinoviSeedance2p5OutputResolution,
 };
 
@@ -32,6 +32,8 @@ pub struct KinoviSeedance2p5DraftState {
   /// all reference videos (each rounded up to a whole second).
   pub total_input_seconds: Option<u8>,
 
+  pub maybe_bitrate: Option<KinoviSeedance2p5Bitrate>,
+
   pub unhandled_request_state: Option<KinoviSeedance2p5RemainingItems>,
 }
 
@@ -111,6 +113,7 @@ impl KinoviSeedance2p5DraftState {
       duration_seconds: self.duration_seconds,
       total_input_seconds: self.total_input_seconds,
       use_face_blur_hack: None,
+      maybe_bitrate: self.maybe_bitrate,
     };
 
     Ok(KinoviSeedance2p5RequestState { request })

crates/api_clients/grok_consumer_client/Cargo.toml

@@ -1,52 +0,0 @@
-[package]
-name = "grok_consumer_client"
-edition = "2024"
-version = "0.0.1"
-publish = false
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-build-acceleration"]
-
-[lib]
-name = "grok_consumer_client"
-path = "src/lib.rs"
-
-[dependencies]
-# Workspace hack (faster compile times)
-workspace-build-acceleration.workspace = true
-
-# Internal
-cloudflare_errors.workspace = true
-cloudflare_mitigation.workspace = true
-cookie_store.workspace = true
-errors.workspace = true
-
-# External
-anyhow.workspace = true
-base64.workspace = true
-bstr.workspace = true
-byteorder.workspace = true
-chrono.workspace = true
-futures.workspace = true
-log.workspace = true
-once_cell.workspace = true
-regex.workspace = true
-scraper = "0.24.0"
-serde.workspace = true
-serde_json.workspace = true
-sha2.workspace = true
-tokio = { version = "1", features = ["full"] }
-url.workspace = true
-uuid.workspace = true
-wreq-util.workspace = true
-wreq.workspace = true
-
-[dev-dependencies]
-filesys.workspace = true
-test_data.workspace = true
-test_utils.workspace = true
-log4rs = "1.4.0"
-env_logger.workspace = true
-env.workspace = true
-chrono.workspace = true
-

crates/api_clients/grok_consumer_client/src/client/browser_user_agents.rs

@@ -1,10 +0,0 @@
-
-/// Firefox 139 (2025) on macOS, which matches wreq's Emulation::Firefox139
-pub const FIREFOX_139_MAC_USER_AGENT : &str =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:139.0) Gecko/20100101 Firefox/139.0";
-
-pub const FIREFOX_143_MAC_USER_AGENT : &str =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0";
-
-pub const FIREFOX_144_MAC_USER_AGENT_UNSUPPORTED_TLS_FINGERPRINTING : &str =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:144.0) Gecko/20100101 Firefox/144.0";

crates/api_clients/grok_consumer_client/src/client/mod.rs

@@ -1 +0,0 @@
-pub mod browser_user_agents;

crates/api_clients/grok_consumer_client/src/credentials/_metadata/crypto_signature/xStatsigIDGenerator.py

@@ -1,517 +0,0 @@
-#!/usr/bin/env python3
-# From https://github.com/kbtit25/grok2api/blob/73c917b15438e12699531ccf9411d3a7d1110cbf/xStatsigIDGenerator.py
-# -*- coding: utf-8 -*-
-
-import base64
-import struct
-import hashlib
-import time
-import secrets
-import requests
-import re
-import json
-from typing import Optional, Dict, Any
-
-class XStatsigIDGenerator:
-    """x-statsig-id 生成器"""
-
-    def __init__(self):
-        self.base_timestamp = int(time.time())  # 使用当前系统时间
-        self.grok_url = "https://grok.com"
-
-    def get_grok_meta_content(self) -> bytes:
-        """
-        从 grok.com 获取 meta 标签中的 grok-site-verification 内容
-        使用多种方法彻底解决403问题
-
-        Returns:
-            48字节的meta内容
-        """
-        print("🌐 正在请求 grok.com...")
-
-        # 定义多种绕过策略
-        strategies = [
-            #self._try_curl_with_proxy,
-            self._try_curl_with_different_ua,
-            #self._try_requests_with_session,
-            #self._try_curl_cffi_advanced,
-            #self._try_alternative_endpoints,
-            #self._try_cached_content
-        ]
-
-        for i, strategy in enumerate(strategies):
-            try:
-                print(f"   尝试策略 {i+1}: {strategy.__name__}")
-                result = strategy()
-                if result:
-                    return result
-            except Exception as e:
-                print(f"   策略 {i+1} 失败: {e}")
-                continue
-
-        # 所有策略都失败，使用备用内容
-        print("   ❌ 所有策略都失败，使用备用meta内容")
-        fallback = b"backup-grok-meta-content-when-request-fails-ok"
-        return fallback + b'\x00' * (48 - len(fallback))
-
-    def _try_curl_with_proxy(self) -> bytes:
-        """策略1: 使用curl + 代理"""
-        import subprocess
-
-        # 尝试多个公共代理
-        proxies = [
-            None,  # 无代理
-            "socks5://127.0.0.1:1080",  # 本地代理
-            "http://127.0.0.1:8080",   # 本地HTTP代理
-        ]
-
-        for proxy in proxies:
-            try:
-                curl_command = [
-                    'curl', '-s', '-L', '--max-time', '15',
-                    '--user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-                    '--header', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                    '--header', 'Accept-Language: en-US,en;q=0.5',
-                    '--header', 'Accept-Encoding: gzip, deflate',
-                    '--header', 'Connection: keep-alive',
-                    '--header', 'Upgrade-Insecure-Requests: 1',
-                    '--compressed'
-                ]
-
-                if proxy:
-                    curl_command.extend(['--proxy', proxy])
-
-                curl_command.append(self.grok_url)
-
-                result = subprocess.run(curl_command, capture_output=True, text=True, timeout=15)
-
-                if result.returncode == 0 and len(result.stdout) > 1000:
-                    print(f"      ✅ curl成功 (代理: {proxy or '无'})")
-                    return self._extract_meta_from_html(result.stdout)
-
-            except Exception:
-                continue
-
-        return None
-
-    def _try_curl_with_different_ua(self) -> bytes:
-        """策略2: 使用不同的User-Agent"""
-        import subprocess
-
-        user_agents = [
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0',
-            #'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
-            #'Mozilla/5.0 (Android 13; Mobile; rv:109.0) Gecko/109.0 Firefox/119.0',
-            #'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-            #'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
-        ]
-
-        for ua in user_agents:
-            try:
-                curl_command = [
-                    'curl-impersonate-ff', '-s', '-L', '--max-time', '10',
-                    '--user-agent', ua,
-                    '--header', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                    '--header', 'Accept-Language: en-US,en;q=0.9',
-                    '--header', 'Cache-Control: no-cache',
-                    '--header', 'Pragma: no-cache',
-                    '--compressed',
-                    self.grok_url
-                ]
-
-                result = subprocess.run(curl_command, capture_output=True, text=True, timeout=10)
-
-                if result.returncode == 0 and len(result.stdout) > 1000:
-                    print(f"      ✅ 不同UA成功")
-                    return self._extract_meta_from_html(result.stdout)
-
-            except Exception:
-                continue
-
-        return None
-
-    def _try_requests_with_session(self) -> bytes:
-        """策略3: 使用requests session模拟真实浏览器行为"""
-        import requests
-        from requests.adapters import HTTPAdapter
-        from urllib3.util.retry import Retry
-
-        session = requests.Session()
-
-        # 配置重试策略
-        retry_strategy = Retry(
-            total=3,
-            backoff_factor=1,
-            status_forcelist=[429, 500, 502, 503, 504],
-        )
-        adapter = HTTPAdapter(max_retries=retry_strategy)
-        session.mount("http://", adapter)
-        session.mount("https://", adapter)
-
-        # 模拟真实浏览器行为
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.9',
-            'Accept-Encoding': 'gzip, deflate, br',
-            'DNT': '1',
-            'Connection': 'keep-alive',
-            'Upgrade-Insecure-Requests': '1',
-            'Sec-Fetch-Dest': 'document',
-            'Sec-Fetch-Mode': 'navigate',
-            'Sec-Fetch-Site': 'none',
-            'Sec-Fetch-User': '?1',
-            'Cache-Control': 'max-age=0'
-        }
-
-        try:
-            # 先访问主页建立session
-            session.get('https://x.com', headers=headers, timeout=5)
-
-            # 再访问grok
-            response = session.get(self.grok_url, headers=headers, timeout=10)
-
-            if response.status_code == 200 and len(response.text) > 1000:
-                print(f"      ✅ session请求成功")
-                return self._extract_meta_from_html(response.text)
-
-        except Exception:
-            pass
-
-        return None
-
-    def _try_curl_cffi_advanced(self) -> bytes:
-        """策略4: 使用curl_cffi高级模拟"""
-        try:
-            from curl_cffi import requests as curl_requests
-
-            # 尝试不同的浏览器模拟
-            impersonations = ['chrome120', 'chrome119', 'safari17', 'firefox119']
-
-            for imp in impersonations:
-                try:
-                    response = curl_requests.get(
-                        self.grok_url,
-                        impersonate=imp,
-                        timeout=10,
-                        headers={
-                            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                            'Accept-Language': 'en-US,en;q=0.9',
-                            'Cache-Control': 'no-cache'
-                        }
-                    )
-
-                    if response.status_code == 200 and len(response.text) > 1000:
-                        print(f"      ✅ curl_cffi成功 ({imp})")
-                        return self._extract_meta_from_html(response.text)
-
-                except Exception:
-                    continue
-
-        except ImportError:
-            pass
-
-        return None
-
-    def _try_alternative_endpoints(self) -> bytes:
-        """策略5: 尝试替代端点"""
-        import subprocess
-
-        # 尝试不同的URL路径
-        alternative_urls = [
-            'https://grok.com/',
-            'https://grok.com/login',
-            'https://grok.com/home',
-            'https://x.com/i/grok'
-        ]
-
-        for url in alternative_urls:
-            try:
-                curl_command = [
-                    'curl', '-s', '-L', '--max-time', '8',
-                    '--user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
-                    '--header', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-                    '--compressed',
-                    url
-                ]
-
-                result = subprocess.run(curl_command, capture_output=True, text=True, timeout=8)
-
-                if result.returncode == 0 and len(result.stdout) > 500:
-                    print(f"      ✅ 替代端点成功: {url}")
-                    meta_result = self._extract_meta_from_html(result.stdout)
-                    if meta_result:
-                        return meta_result
-
-            except Exception:
-                continue
-
-        return None
-
-    def _try_cached_content(self) -> bytes:
-        """策略6: 使用缓存或预设内容"""
-        # 如果有真实的grok meta内容，可以在这里硬编码
-        known_meta_contents = [
-            "grok-site-verification-content-2024-production-v1",
-            "x-grok-verification-meta-content-stable-release",
-            "grok-meta-verification-string-for-api-access"
-        ]
-
-        for content in known_meta_contents:
-            try:
-                meta_bytes = content.encode('utf-8')
-                if len(meta_bytes) < 48:
-                    meta_bytes = meta_bytes + b'\x00' * (48 - len(meta_bytes))
-                elif len(meta_bytes) > 48:
-                    meta_bytes = meta_bytes[:48]
-
-                print(f"      ✅ 使用预设内容: {content[:30]}...")
-                return meta_bytes
-
-            except Exception:
-                continue
-
-        return None
-
-    def _extract_meta_from_html(self, html_content: str) -> bytes:
-        """从HTML中提取meta内容"""
-        patterns = [
-            r'<meta\s+name=["\']grok-site-verification["\']\s+content=["\']([^"\']+)["\']',
-            r'<meta\s+content=["\']([^"\']+)["\']\s+name=["\']grok-site-verification["\']',
-            r'grok-site-verification["\']?\s*(?:content|value)\s*=\s*["\']([^"\']+)["\']',
-            # 扩展模式，查找其他可能的meta标签
-            r'<meta\s+name=["\']verification["\']\s+content=["\']([^"\']+)["\']',
-            r'<meta\s+name=["\']site-verification["\']\s+content=["\']([^"\']+)["\']'
-        ]
-
-        for pattern in patterns:
-            match = re.search(pattern, html_content, re.IGNORECASE)
-            if match:
-                verification_content = match.group(1)
-                print(f"      ✅ 找到meta内容: {verification_content[:50]}...")
-
-                # 转换为48字节
-                meta_bytes = verification_content.encode('utf-8')
-                if len(meta_bytes) < 48:
-                    meta_bytes = meta_bytes + b'\x00' * (48 - len(meta_bytes))
-                elif len(meta_bytes) > 48:
-                    meta_bytes = meta_bytes[:48]
-
-                return meta_bytes
-
-        # 如果没找到特定的meta标签，尝试从HTML中提取其他有用信息
-        if 'grok' in html_content.lower() and len(html_content) > 1000:
-            # 使用HTML内容的哈希作为meta内容
-            import hashlib
-            content_hash = hashlib.sha256(html_content.encode()).hexdigest()[:48]
-            meta_bytes = content_hash.encode('utf-8')
-            if len(meta_bytes) < 48:
-                meta_bytes = meta_bytes + b'\x00' * (48 - len(meta_bytes))
-
-            print(f"      ✅ 使用内容哈希作为meta: {content_hash[:32]}...")
-            return meta_bytes
-
-        return None
-
-    def generate_browser_fingerprint(self) -> str:
-        """
-        生成浏览器指纹信息 (结合方法1和方法3)
-
-        Returns:
-            指纹字符串
-        """
-        print("🔍 生成浏览器指纹...")
-
-        # 模拟浏览器指纹信息
-        fingerprint_data = {
-            "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",
-            "language": "en",
-            "languages": ["en", "en-US"],
-            "platform": "MacIntel",
-            "cookieEnabled": True,
-            "doNotTrack": None,
-            "screenWidth": 450,
-            "screenHeight": 654,
-            "screenColorDepth": 24,
-            "screenPixelDepth": 24,
-            "screenAvailWidth": 450,
-            "screenAvailHeight": 654,
-            "innerWidth": 450,
-            "innerHeight": 654,
-            "outerWidth": 1920,
-            "outerHeight": 1055,
-            "timezone": "America/New York",
-            "timezoneOffset": -400,
-            "hardwareConcurrency": 14,
-            "deviceMemory": 16,
-            "maxTouchPoints": 0
-        }
-
-        # 方法3: 生成指纹哈希
-        fingerprint_string = json.dumps(fingerprint_data, sort_keys=True, separators=(',', ':'))
-        fingerprint_hash = hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
-
-        print(f"   指纹数据长度: {len(fingerprint_string)} 字符")
-        print(f"   指纹哈希: {fingerprint_hash[:32]}...")
-
-        return fingerprint_hash
-
-    def generate_x_statsig_id(self, method: str = "GET", pathname: str = "/") -> str:
-        """
-        生成完整的 x-statsig-id
-
-        Args:
-            method: 请求方式 (GET/POST)
-            pathname: 请求路径
-
-        Returns:
-            生成的 x-statsig-id 字符串
-        """
-        print("=" * 60)
-        print("🚀 开始生成 x-statsig-id")
-        print("=" * 60)
-
-        print(f"📋 生成参数:")
-        print(f"   Method: {method}")
-        print(f"   Pathname: {pathname}")
-
-        # 1. 获取 grok.com 的 meta content
-        meta_content = self.get_grok_meta_content()
-
-        # 2. 生成浏览器指纹
-        fingerprint = self.generate_browser_fingerprint()
-
-        # 3. 生成当前时间戳
-        current_timestamp = int(time.time())
-        relative_timestamp = current_timestamp - self.base_timestamp
-
-        print(f"⏰ 时间信息:")
-        print(f"   当前时间戳: {current_timestamp}")
-        print(f"   基准时间戳: {self.base_timestamp}")
-        print(f"   相对时间戳: {relative_timestamp}")
-
-        # 4. 生成时间戳字节 (小端序)
-        timestamp_bytes = struct.pack('<I', relative_timestamp)
-        print(f"   时间戳字节: {timestamp_bytes.hex()}")
-
-        # 5. 生成SHA256
-        sha_input = f"{method}!{pathname}!{relative_timestamp}{fingerprint}"
-        sha256_hash = hashlib.sha256(sha_input.encode('utf-8')).digest()
-        sha256_16bytes = sha256_hash[:16]
-
-        print(f"🔐 SHA256信息:")
-        print(f"   输入字符串: {sha_input[:100]}...")
-        print(f"   SHA256前16字节: {sha256_16bytes.hex()}")
-
-        # 6. 固定值
-        fixed_byte = b'\x03'
-
-        # 7. 组合payload数据
-        payload_data = meta_content + timestamp_bytes + sha256_16bytes + fixed_byte
-        print(f"📦 Payload长度: {len(payload_data)} 字节")
-
-        # 8. 生成异或key并加密
-        xor_key = secrets.randbits(8)
-        encrypted_payload = bytes([b ^ xor_key for b in payload_data])
-
-        print(f"🔑 异或信息:")
-        print(f"   异或key: 0x{xor_key:02x} ({xor_key})")
-
-        # 9. 组合最终数据
-        final_data = bytes([xor_key]) + encrypted_payload
-        print(f"   最终数据长度: {len(final_data)} 字节")
-
-        # 10. Base64编码
-        result = base64.b64encode(final_data).decode('utf-8')
-
-        print(f"✅ 生成结果:")
-        print(f"   x-statsig-id: {result}")
-        print(f"   长度: {len(result)} 字符")
-
-        return result
-
-    def verify_generated_id(self, statsig_id: str) -> bool:
-        """
-        验证生成的ID结构是否正确
-
-        Args:
-            statsig_id: 要验证的ID
-
-        Returns:
-            验证是否通过
-        """
-        print("\n" + "=" * 60)
-        print("🔍 验证生成的ID结构")
-        print("=" * 60)
-
-        try:
-            # Base64解码
-            decoded_bytes = base64.b64decode(statsig_id)
-            print(f"✅ Base64解码成功，长度: {len(decoded_bytes)} 字节")
-
-            # 提取异或key
-            xor_key = decoded_bytes[0]
-            print(f"✅ 异或key: 0x{xor_key:02x} ({xor_key})")
-
-            # 异或解密
-            decrypted = bytearray()
-            for i in range(1, len(decoded_bytes)):
-                decrypted.append(decoded_bytes[i] ^ xor_key)
-
-            print(f"✅ 解密后长度: {len(decrypted)} 字节")
-
-            # 验证数据结构
-            expected_length = 48 + 4 + 16 + 1  # meta + timestamp + sha256 + fixed
-            if len(decrypted) == expected_length:
-                print(f"✅ 数据长度正确: {len(decrypted)}/{expected_length}")
-            else:
-                print(f"❌ 数据长度错误: {len(decrypted)}/{expected_length}")
-                return False
-
-            # 检查固定值
-            fixed_val = decrypted[-1]
-            if fixed_val == 3:
-                print(f"✅ 固定值正确: {fixed_val}")
-            else:
-                print(f"❌ 固定值错误: {fixed_val} (期望: 3)")
-                return False
-
-            # 解析时间戳
-            timestamp_bytes = decrypted[48:52]
-            timestamp = struct.unpack('<I', timestamp_bytes)[0]
-            actual_time = self.base_timestamp + timestamp
-
-            print(f"✅ 时间戳解析:")
-            print(f"   相对时间: {timestamp} 秒")
-            print(f"   绝对时间: {actual_time}")
-            print(f"   时间差: {abs(time.time() - actual_time):.1f} 秒")
-
-            print("🎉 ID结构验证通过！")
-            return True
-
-        except Exception as e:
-            print(f"❌ 验证失败: {e}")
-            return False
-
-def main():
-    """主函数 - 演示完整流程"""
-    generator = XStatsigIDGenerator()
-
-    # 生成 x-statsig-id
-    method = "POST"
-    pathname = "/rest/app-chat/conversations/new"
-
-    statsig_id = generator.generate_x_statsig_id(method, pathname)
-    print("Generated id", statsig_id)
-
-    # 验证生成的ID
-    generator.verify_generated_id(statsig_id)
-
-    print(f"\n" + "=" * 60)
-    print("📋 使用说明")
-    print("=" * 60)
-    print("在HTTP请求中使用:")
-    print(f"Headers: {{'x-statsig-id': '{statsig_id}'}}")
-
-if __name__ == "__main__":
-    main()

crates/api_clients/grok_consumer_client/src/credentials/_metadata/crypto_signature/xctid.py

@@ -1,193 +0,0 @@
-#!/usr/bin/env python3
-
-# From https://github.com/realasfngl/Grok-Api/blob/d8f7c6754d95ad3707b214f93930ee454724035d/core/xctid.py
-
-from math      import floor, copysign, pi, cos, sin
-from base64    import b64decode, b64encode
-from re        import findall, sub
-from typing    import List, Dict
-from random    import random
-from hashlib   import sha256
-from struct    import pack
-from time      import time
-
-
-class Signature:
-
-
-    @staticmethod
-    def _h(x: float, _param: float, c: float, e: bool):
-        f = ((x * (c - _param)) / 255.0) + _param
-        if e:
-            return floor(f)
-        rounded = round(float(f), 2)
-        if rounded == 0.0:
-            return 0.0
-        return rounded
-
-    @staticmethod
-    def cubicBezierEased(t: float, x1: float, y1: float, x2: float, y2: float) -> float:
-        def bezier(u: float):
-            omu = 1.0 - u
-            b1 = 3.0 * omu * omu * u
-            b2 = 3.0 * omu * u * u
-            b3 = u * u * u
-            x = b1 * x1 + b2 * x2 + b3
-            y = b1 * y1 + b2 * y2 + b3
-            return x, y
-
-        lo, hi = 0.0, 1.0
-        for _ in range(80):
-            mid = 0.5 * (lo + hi)
-            if bezier(mid)[0] < t:
-                lo = mid
-            else:
-                hi = mid
-        u = 0.5 * (lo + hi)
-        return bezier(u)[1]
-
-    @staticmethod
-    def xa(svg: str) -> List[List[int]]:
-        s = (svg)
-        substr = s[9:]
-        parts = substr.split("C")
-        out = []
-        for part in parts:
-            cleaned = sub(r"[^\d]+", " ", part).strip()
-            if cleaned == "":
-                nums = [0]
-            else:
-                nums = [int(tok) for tok in cleaned.split() if tok != ""]
-            out.append(nums)
-        return out
-
-    @staticmethod
-    def tohex(num: float) -> str:
-        rounded = round(float(num), 2)
-        if rounded == 0.0:
-            return "0"
-        sign = "-" if copysign(1.0, rounded) < 0 else ""
-        absval = abs(rounded)
-        intpart = int(floor(absval))
-        frac = absval - intpart
-        if frac == 0.0:
-            return sign + format(intpart, "x")
-        frac_digits = []
-        f = frac
-        for _ in range(20):
-            f *= 16
-            digit = int(floor(f + 1e-12))
-            frac_digits.append(format(digit, "x"))
-            f -= digit
-            if abs(f) < 1e-12:
-                break
-        frac_str = "".join(frac_digits).rstrip("0")
-        if frac_str == "":
-            return sign + format(intpart, "x")
-        return sign + format(intpart, "x") + "." + frac_str
-
-    @staticmethod
-    def simulateStyle(values: List[int], c: int) -> Dict[str,str]:
-        duration = 4096
-        currentTime = round(c / 10.0) * 10
-        t = currentTime / duration
-
-        cp = [Signature._h(v, -1 if (i % 2) else 0, 1, False) for i, v in enumerate(values[7:])]
-
-        easedY = Signature.cubicBezierEased(t, cp[0], cp[1], cp[2], cp[3])
-
-        start = [float(x) for x in values[0:3]]
-        end = [float(x) for x in values[3:6]]
-        r = round(start[0] + (end[0] - start[0]) * easedY)
-        g = round(start[1] + (end[1] - start[1]) * easedY)
-        b = round(start[2] + (end[2] - start[2]) * easedY)
-        color = f"rgb({r}, {g}, {b})"
-
-        endAngle = Signature._h(values[6], 60, 360, True)
-        angle = endAngle * easedY
-        rad = angle * pi / 180.0
-
-        def is_effectively_zero(val: float) -> bool:
-            return abs(val) < 1e-7
-
-        def is_effectively_integer(val: float) -> bool:
-            return abs(val - round(val)) < 1e-7
-
-        cosv = cos(rad)
-        sinv = sin(rad)
-
-        if is_effectively_zero(cosv):
-            a = 0
-            d = 0
-        else:
-            if is_effectively_integer(cosv):
-                a = int(round(cosv))
-                d = int(round(cosv))
-            else:
-                a = f"{cosv:.6f}"
-                d = f"{cosv:.6f}"
-
-        if is_effectively_zero(sinv):
-            bval = 0
-            cval = 0
-        else:
-            if is_effectively_integer(sinv):
-                bval = int(round(sinv))
-                cval = int(round(-sinv))
-            else:
-                bval = f"{sinv:.7f}"
-                cval = f"{(-sinv):.7f}"
-
-        transform = f"matrix({a}, {bval}, {cval}, {d}, 0, 0)"
-        return {"color": color, "transform": transform}
-
-    @staticmethod
-    def xs(x_bytes: bytes, svg: str, x_values: list) -> str:
-        arr = list(x_bytes)
-        idx = arr[x_values[0]] % 16
-        c = ((arr[x_values[1]] % 16) * (arr[x_values[2]] % 16)) * (arr[x_values[3]] % 16)
-        o = Signature.xa(svg)
-        vals = o[idx]
-        k = Signature.simulateStyle(vals, c)
-
-        concat = str(k["color"]) + str(k["transform"])
-        matches = findall(r"[\d\.\-]+", concat)
-        converted = []
-        for m in matches:
-            num = float(m)
-            hexstr = Signature.tohex(num)
-            converted.append(hexstr)
-        joined = "".join(converted)
-        cleaned = joined.replace(".", "").replace("-", "")
-        return cleaned
-
-    @staticmethod
-    def generate_sign(path: str, method: str, verification: str, svg: str, x_values: list, time_n: int = None, random_float: float = None) -> str:
-
-        n = int(time() - 1682924400) if not time_n else time_n
-        t = pack('<I', n)
-        r = b64decode(verification)
-        o = Signature.xs(r, svg, x_values)
-
-        msg = "!".join([method, path, str(n)]) + "obfiowerehiring" + o
-        digest = sha256(msg.encode('utf-8')).digest()[:16]
-
-        prefix_byte = int(floor(random() if not random_float else random_float * 256))
-        assembled = bytes([prefix_byte]) + r + t + digest + bytes([3])
-
-        arr = bytearray(assembled)
-        if len(arr) > 0:
-            first = arr[0]
-            for i in range(1, len(arr)):
-                arr[i] = arr[i] ^ first
-
-        return b64encode(bytes(arr)).decode('ascii').replace('=', '')
-
-
-def main():
-    sig = Signature.generate_sign('/rest/app-chat/conversations/new', 'POST', self.verification_token, self.svg_data, self.numbers)
-    print('Signature', sig)
-    pass
-
-if __name__ == 'main':
-    main()

crates/api_clients/grok_consumer_client/src/credentials/grok_client_secrets.rs

@@ -1,36 +0,0 @@
-use crate::datatypes::api::baggage::Baggage;
-use crate::datatypes::api::sentry_trace::SentryTrace;
-use crate::datatypes::api::svg_path_data::SvgPathData;
-use crate::datatypes::api::user_email::UserEmail;
-use crate::datatypes::api::user_id::UserId;
-use crate::datatypes::api::verification_token::VerificationToken;
-use crate::datatypes::api::xsid_numbers::XsidNumbers;
-
-#[derive(Clone)]
-pub struct GrokClientSecrets {
-  /// Baggage is <meta> tag data from the index.html
-  pub baggage: Baggage,
-
-  /// Baggage is <meta> tag data from the index.html
-  pub sentry_trace: SentryTrace,
-
-  /// Baggage is <meta> tag data from the index.html
-  pub verification_token: VerificationToken,
-
-  /// The SVG path data to use (chosen via verification_token)
-  /// We selected one of four-ish possible SVG paths (of length >= 200) by
-  /// the `verification token -> animation index` algo.
-  pub svg_path_data: SvgPathData,
-
-  /// This doesn't come from the index.html
-  /// These numbers are separately loaded from the xsid javascript
-  pub numbers: XsidNumbers,
-
-  /// From index.html
-  /// Not strictly needed to sign requests, but typically needed to generate URLs.
-  pub user_id: UserId,
-
-  /// From index.html
-  /// Not needed, but returned alongside other details.
-  pub user_email: Option<UserEmail>,
-}

crates/api_clients/grok_consumer_client/src/credentials/grok_cookies.rs

@@ -1,24 +0,0 @@
-
-#[derive(Clone)]
-pub struct GrokCookies {
-  /// The entire cookies header (for now)
-  cookies: String,
-}
-
-impl GrokCookies {
-  pub fn new(cookies: String) -> Self {
-    Self { cookies }
-  }
-
-  pub fn as_str(&self) -> &str {
-    &self.cookies
-  }
-
-  pub fn as_bytes(&self) -> &[u8] {
-    self.cookies.as_bytes()
-  }
-
-  pub fn to_string(&self) -> String {
-    self.cookies.clone()
-  }
-}

crates/api_clients/grok_consumer_client/src/credentials/grok_full_credentials.rs

@@ -1,30 +0,0 @@
-use crate::credentials::grok_client_secrets::GrokClientSecrets;
-use crate::credentials::grok_cookies::GrokCookies;
-use crate::datatypes::api::user_id::UserId;
-
-/// Includes the cookies and all the magic other bits needed to call the API.
-#[derive(Clone)]
-pub struct GrokFullCredentials {
-  /// Entire cookie payload
-  pub cookies: GrokCookies,
-  
-  /// Secrets from `index.html` and the xsid javascript.
-  /// Required for video generation.
-  pub client_secrets: GrokClientSecrets,
-}
-
-impl GrokFullCredentials {
-  pub fn from_cookies_and_client_secrets(
-    cookies: GrokCookies, 
-    client_secrets: GrokClientSecrets
-  ) -> Self {
-    Self {
-      cookies,
-      client_secrets,
-    }
-  }
-  
-  pub fn get_user_id_ref(&self) -> &UserId {
-    &self.client_secrets.user_id
-  }
-}

crates/api_clients/grok_consumer_client/src/credentials/grok_user_data.rs

@@ -1,22 +0,0 @@
-use crate::datatypes::api::user_email::UserEmail;
-use crate::datatypes::api::user_id::UserId;
-
-#[derive(Clone)]
-pub struct GrokUserData {
-  /// From index.html
-  /// Not strictly needed to sign requests, but typically needed to generate URLs.
-  pub user_id: UserId,
-
-  /// From index.html
-  /// Not needed, but returned alongside other details.
-  pub user_email: Option<UserEmail>,
-}
-
-impl GrokUserData {
-  pub fn from_id_and_email(id: String, email: Option<String>) -> Self {
-    Self {
-      user_id: UserId(id),
-      user_email: email.map(|email| UserEmail(email)),
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/credentials/mod.rs

@@ -1,4 +0,0 @@
-pub mod grok_client_secrets;
-pub mod grok_cookies;
-pub mod grok_full_credentials;
-pub mod grok_user_data;

crates/api_clients/grok_consumer_client/src/datatypes/api/aspect_ratio.rs

@@ -1,13 +0,0 @@
-use serde::Serialize;
-
-#[derive(Serialize, Clone, Copy, Debug)]
-pub enum AspectRatio {
-  #[serde(rename = "2:3")]
-  TallTwoByThree,
-
-  #[serde(rename = "3:2")]
-  WideThreeByTwo,
-
-  #[serde(rename = "1:1")]
-  Square,
-}

crates/api_clients/grok_consumer_client/src/datatypes/api/baggage.rs

@@ -1,4 +0,0 @@
-
-/// "baggage" data from the <meta> tags
-#[derive(Debug, Clone)]
-pub struct Baggage(pub(crate) String);

crates/api_clients/grok_consumer_client/src/datatypes/api/file_id.rs

@@ -1,5 +0,0 @@
-
-/// Type for File IDs. 
-/// File IDs are UUIds. 
-#[derive(Clone, Debug)]
-pub struct FileId(pub String);

crates/api_clients/grok_consumer_client/src/datatypes/api/mod.rs

@@ -1,12 +0,0 @@
-pub mod aspect_ratio;
-pub mod baggage;
-pub mod file_id;
-pub mod video_generation_mode;
-pub mod post_id;
-pub mod request_id;
-pub mod sentry_trace;
-pub mod svg_path_data;
-pub mod user_email;
-pub mod user_id;
-pub mod verification_token;
-pub mod xsid_numbers;

crates/api_clients/grok_consumer_client/src/datatypes/api/post_id.rs

@@ -1,12 +0,0 @@
-
-/// Type for Post IDs.
-/// Post IDs are UUIds.
-/// These are often shared with media/image uploads' `FileId`s, but seldom video generation `FileId`s
-#[derive(Clone, Debug)]
-pub struct PostId(pub String);
-
-impl PostId {
-  pub fn to_string(&self) -> String {
-    self.0.clone()
-  }
-}

crates/api_clients/grok_consumer_client/src/datatypes/api/request_id.rs

@@ -1,13 +0,0 @@
-
-/// Request IDs. 
-/// Type for image websocket "tasks" 
-/// These are for image generation websockets.
-/// Request IDs are UUIds. 
-#[derive(Clone, Debug)]
-pub struct RequestId(pub String);
-
-impl RequestId {
-  pub fn to_string(&self) -> String {
-    self.0.clone()
-  }
-}

crates/api_clients/grok_consumer_client/src/datatypes/api/sentry_trace.rs

@@ -1,40 +0,0 @@
-use uuid::Uuid;
-
-/// "sentry trace" data from the <meta> tags
-#[derive(Debug, Clone)]
-pub struct SentryTrace(pub (crate) String);
-
-impl SentryTrace {
-
-  /// Generate the format of trace sent in the HTTP request headers
-  pub fn to_http_request_header(&self) -> String {
-    // Python reference:
-    //   f'{self.sentry_trace}-{str(uuid4()).replace("-", "")[:16]}-0',
-    // Inner bit:
-    //   str(uuid4()).replace("-", "")[:16]
-    let sentry_start = &self.0;
-    let sentry_uuid = Uuid::new_v4().to_string();
-    let sentry_inner = sentry_uuid.replace("-", "")[..16].to_string();
-
-    format!("{sentry_start}-{sentry_inner}-0")
-  }
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::datatypes::api::sentry_trace::SentryTrace;
-
-  #[test]
-  fn test_real_trace() {
-    let short = SentryTrace("d6d7c55e4a489c0dabaed06e5c7b257b".to_string());
-    let header = short.to_http_request_header();
-    assert_eq!(header.len(), 51);
-  }
-
-  #[test]
-  fn test_invalid_trace() {
-    let short = SentryTrace("".to_string());
-    let header = short.to_http_request_header();
-    assert_eq!(header.len(), 19); // header is invalid, we're just making sure Rust doesn't panic
-  }
-}

crates/api_clients/grok_consumer_client/src/datatypes/api/svg_path_data.rs

@@ -1,4 +0,0 @@
-
-/// Data on an SVG path, eg. <svg d={data} />
-#[derive(Debug, Clone)]
-pub struct SvgPathData(pub(crate) String);

crates/api_clients/grok_consumer_client/src/datatypes/api/user_email.rs

@@ -1,9 +0,0 @@
-/// Type for User Emails
-#[derive(Clone, Debug)]
-pub struct UserEmail(pub String);
-
-impl UserEmail {
-  pub fn to_string(&self) -> String {
-    self.0.clone()
-  }
-}

crates/api_clients/grok_consumer_client/src/datatypes/api/user_id.rs

@@ -1,12 +0,0 @@
-
-/// Type for User IDs.
-/// User IDs are UUIDs.
-#[derive(Clone, Debug)]
-pub struct UserId(pub String);
-
-impl UserId {
-  pub fn to_string(&self) -> String {
-    self.0.clone()
-  }
-}
-

crates/api_clients/grok_consumer_client/src/datatypes/api/verification_token.rs

@@ -1,3 +0,0 @@
-
-#[derive(Debug, Clone)]
-pub struct VerificationToken(pub(crate) String);

crates/api_clients/grok_consumer_client/src/datatypes/api/video_generation_mode.rs

@@ -1,27 +0,0 @@
-
-/// The video generation mode selector.
-#[derive(Copy, Clone, Debug)]
-pub enum VideoGenerationMode {
-  /// Supposedly the "NSFW" mode.
-  Spicy,
-
-  Fun,
-
-  Normal,
-
-  /// This might be the only mode that allows for a custom text prompt
-  Custom,
-}
-
-impl VideoGenerationMode {
-
-  /// This is the value for `--mode` in video generation chats.
-  pub fn as_api_mode_arg(&self) -> &'static str {
-    match self {
-      Self::Spicy => "extremely-spicy-or-crazy",
-      Self::Fun => "extremely-crazy",
-      Self::Normal => "normal",
-      Self::Custom => "custom",
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/datatypes/api/xsid_numbers.rs

@@ -1,6 +0,0 @@
-
-/// Magic numbers pulled from Javascript
-#[derive(Clone, Debug)]
-pub struct XsidNumbers {
-  pub numbers: Vec<u32>,
-}

crates/api_clients/grok_consumer_client/src/datatypes/file_upload_spec.rs

@@ -1,18 +0,0 @@
-use std::path::Path;
-
-/// Descriptor of a file to be uploaded
-/// Includes necessary metadata
-pub enum FileUploadSpec<P: AsRef<Path>> {
-  /// Path to the file
-  Path(P),
-
-  /// Bytes of the file
-  Bytes {
-    /// Must be owned.
-    bytes: Vec<u8>,
-    /// Must be owned.
-    filename: String,
-    /// Mime type
-    mimetype: String,
-  }
-}

crates/api_clients/grok_consumer_client/src/datatypes/mod.rs

@@ -1,2 +0,0 @@
-pub mod api;
-pub mod file_upload_spec;

crates/api_clients/grok_consumer_client/src/error/categorize_grok_http_error.rs

@@ -1,37 +0,0 @@
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::error::grok_specific_api_error::GrokSpecificApiError;
-use cloudflare_errors::filter_cloudflare_errors::filter_cloudflare_errors;
-use wreq::StatusCode;
-
-/// We know the response is an error at this point.
-/// We're just turning it into the right error.
-pub fn categorize_grok_http_error(status_code: StatusCode, maybe_body: Option<&str>) -> GrokError {
-
-  if let Some(body) = maybe_body {
-    if let Err(err) = filter_cloudflare_errors(status_code.as_u16(), body) {
-      return GrokGenericApiError::CloudflareError(err).into();
-    }
-
-    let body = body.to_lowercase();
-    let anti_bot = body.contains("anti-bot") || body.contains("rejected");
-
-    if anti_bot {
-      return GrokSpecificApiError::AutomationBlocked.into();
-    }
-
-    match status_code {
-      StatusCode::TOO_MANY_REQUESTS => {
-        return GrokSpecificApiError::TooManyVideos.into();
-      }
-      _ => {},
-    }
-
-    return GrokGenericApiError::UncategorizedBadResponseWithStatusAndBody {
-      status_code,
-      body: body.to_string(),
-    }.into();
-  }
-
-  GrokGenericApiError::UncategorizedBadResponseWithStatus(status_code).into()
-}

crates/api_clients/grok_consumer_client/src/error/grok_client_error.rs

@@ -1,154 +0,0 @@
-use base64::DecodeError;
-use std::error::Error;
-use std::fmt::{Display, Formatter};
-
-#[derive(Debug)]
-pub enum GrokClientError {
-
-  /// An error was encountered in building the Wreq client
-  WreqClientError(wreq::Error),
-
-  /// Error serializing a message to send the websocket
-  WebsocketRequestSerializationError(serde_json::Error),
-
-  /// Error locking the websocket for sending/receiving
-  WebsocketLockError,
-
-  /// Error reading from a websocket.
-  WebsocketReadError(wreq::Error),
-
-  /// Error sending to an open websocket.
-  WebsocketSendError(wreq::Error),
-
-  /// Can't open a local file for uploading.
-  CannotOpenLocalFileForUpload(std::io::Error),
-
-  /// Can't read a local file for uploading.
-  CannotReadLocalFileForUpload(std::io::Error),
-
-  /// Couldn't write to the file
-  CannotOpenLocalFileForWriting(std::io::Error),
-
-  /// Couldn't write to the file
-  CannotWriteLocalFile(std::io::Error),
-
-  /// File for upload has an invalid path.
-  FileForUploadHasInvalidPath,
-
-  /// Error parsing HTML
-  HtmlParsingError,
-  
-  /// Error decoding verification token
-  FailedToDecodeVerificationToken(DecodeError),
-  
-  /// Verification token bytes are invalid
-  InvalidVerificationTokenBytes,
-  
-  /// Our script logic is out of date
-  ScriptLogicOutOfDate,
-
-  /// Our script logic is out of date (SVG)
-  ScriptSvgLogicOutOfDate,
-
-  /// Our script logic is out of date (script 1)
-  Script1LogicOutOfDate,
-
-  /// Our script logic is out of date (script 2)
-  Script2LogicOutOfDate,
-  
-  /// Our script logic is out of date (xsid script)
-  ScriptXsidLogicOutOfDate,
-
-  /// Signature algorithm isn't working with inputs
-  BadSignatureInputs,
-  
-  /// Something is broken with timeout math
-  TimeoutMathBroken,
-
-  /// Can't make request because cookies aren't present
-  NoCookiesPresent,
-
-  /// Unknown error generating video
-  ErrorGeneratingVideo,
-
-//  /// An error reading the file for upload.
-//  FileForUploadReadError(std::io::Error),
-//
-//  /// The file path provided for upload is invalid.
-//  FileForUploadHasInvalidPath,
-//
-//  /// Something is wrong with the JWT bearer token.
-//  /// This error originates on our end as we try to parse the JWT.
-//  LocalJwtClaimsParseError(String),
-//
-//  /// There was an error constructing the form-multipart request.
-//  MultipartFormError(wreq::Error),
-//
-//  /// We haven't received a bearer token yet
-//  /// This is our own internal application state error, not something Sora returns.
-//  /// We know our client can't make the request, so we preemptively fail it.
-//  NoBearerTokenForRequest,
-//
-//  /// A sentinel token is not present in the client, which is required for some requests.
-//  NoSentinelTokenForRequest,
-//
-//  /// Issue with using the SoraCredentialBuilder.
-//  SoraCredentialBuilderError(&'static str),
-//
-//  /// Error parsing a request URL.
-//  UrlParseError(url::ParseError),
-//
-//  /// Error serializing the sentinel store token to JSON (typically for persistent storage).
-//  CouldNotSerializeSentinelTokenStore(serde_json::Error),
-//
-//  /// Error deserializing the sentinel token from JSON (typically for persistent storage).
-//  CouldNotDeserializeSentinelTokenStore {
-//    error: serde_json::Error,
-//    raw_json: String
-//  },
-}
-
-impl Error for GrokClientError {}
-
-impl Display for GrokClientError {
-  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-    match self {
-      Self::WreqClientError(err) => write!(f, "Wreq client error (during client creation): {}", err),
-      Self::WebsocketRequestSerializationError(err) => write!(f, "Websocket request serialization error: {}", err),
-      Self::WebsocketLockError => write!(f, "Websocket lock error"),
-      Self::WebsocketReadError(err) => write!(f, "Websocket read error: {}", err),
-      Self::WebsocketSendError(err) => write!(f, "Websocket send error: {}", err),
-      Self::CannotOpenLocalFileForUpload(err) => write!(f, "Cannot open local file for upload: {}", err),
-      Self::CannotReadLocalFileForUpload(err) => write!(f, "Cannot read local file for upload: {}", err),
-      Self::CannotOpenLocalFileForWriting(err) => write!(f, "Cannot open local file for writing: {}", err),
-      Self::CannotWriteLocalFile(err) => write!(f, "Cannot write local file: {}", err),
-      Self::FileForUploadHasInvalidPath => write!(f, "File for upload has invalid path"),
-      Self::HtmlParsingError => write!(f, "Html parsing error"),
-      Self::FailedToDecodeVerificationToken(err) => write!(f, "Failed to decode verification token: {}", err),
-      Self::InvalidVerificationTokenBytes => write!(f, "Invalid verification token bytes"),
-      Self::ScriptLogicOutOfDate => write!(f, "Script logic out of date"),
-      Self::ScriptSvgLogicOutOfDate => write!(f, "Script logic out of date (SVG parsing)"),
-      Self::Script1LogicOutOfDate => write!(f, "Script logic out of date (script 1)"),
-      Self::Script2LogicOutOfDate => write!(f, "Script logic out of date (script 2)"),
-      Self::ScriptXsidLogicOutOfDate => write!(f, "Script logic out of date (xsid script)"),
-      Self::BadSignatureInputs => write!(f, "Bad signature inputs"),
-      Self::TimeoutMathBroken => write!(f, "Timeout math is broken"),
-      Self::NoCookiesPresent => write!(f, "No cookies present"),
-      Self::ErrorGeneratingVideo => write!(f, "Error generating video"),
-
-      //Self::FileForUploadReadError(err) => write!(f, "Error reading file for upload: {}", err),
-      //Self::FileForUploadHasInvalidPath => write!(f, "The file path provided for upload is invalid."),
-      //Self::LocalJwtClaimsParseError(msg) => write!(f, "Local JWT claims parse error: {}", msg),
-      //Self::MultipartFormError(err) => write!(f, "Multipart form error: {}", err),
-      //Self::NoBearerTokenForRequest => write!(f, "No bearer token available. The client needs a bearer token to make the request."),
-      //Self::NoSentinelTokenForRequest => write!(f, "No sentinel token available. The client needs a sentinel token to make the request."),
-      //Self::SoraCredentialBuilderError(msg) => write!(f, "Sora Credential Builder error: {}", msg),
-      //Self::UrlParseError(err) => write!(f, "URL parse error: {}", err),
-      //Self::CouldNotSerializeSentinelToken(err) => write!(f, "Could not serialize sentinel token to JSON: {}", err),
-      //Self::CouldNotSerializeSentinelTokenStore(err) => write!(f, "Could not serialize sentinel token store to JSON: {}", err),
-      //Self::CouldNotDeserializeSentinelTokenStore { error, raw_json} => {
-      //  write!(f, "Could not deserialize sentinel token store from JSON: {:?}, raw JSON: {}", error, raw_json)
-      //},
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/error/grok_error.rs

@@ -1,53 +0,0 @@
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::error::grok_specific_api_error::GrokSpecificApiError;
-use cloudflare_errors::cloudflare_error::CloudflareError;
-use std::error::Error;
-
-#[derive(Debug)]
-pub enum GrokError {
-  Client(GrokClientError),
-  ApiSpecific(GrokSpecificApiError),
-  ApiGeneric(GrokGenericApiError),
-}
-
-impl GrokError {
-  pub fn is_grok_having_downtime_issues(&self) -> bool {
-    match self {
-      Self::ApiGeneric(GrokGenericApiError::CloudflareError(CloudflareError::BadGateway502)) => true,
-      Self::ApiGeneric(GrokGenericApiError::CloudflareError(CloudflareError::GatewayTimeout504)) => true,
-      Self::ApiGeneric(GrokGenericApiError::CloudflareError(CloudflareError::TimeoutOccurred524)) => true,
-      _ => false,
-    }
-  }
-}
-
-impl Error for GrokError {}
-
-impl std::fmt::Display for GrokError {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    match self {
-      Self::Client(e) => write!(f, "GrokClientError: {:?}", e),
-      Self::ApiSpecific(e) => write!(f, "GrokSpecificApiError: {:?}", e),
-      Self::ApiGeneric(e) => write!(f, "GrokGenericApiError: {:?}", e),
-    }
-  }
-}
-
-impl From<GrokClientError> for GrokError {
-  fn from(error: GrokClientError) -> Self {
-    Self::Client(error)
-  }
-}
-
-impl From<GrokSpecificApiError> for GrokError {
-  fn from(error: GrokSpecificApiError) -> Self {
-    Self::ApiSpecific(error)
-  }
-}
-
-impl From<GrokGenericApiError> for GrokError {
-  fn from(error: GrokGenericApiError) -> Self {
-    Self::ApiGeneric(error)
-  }
-}

crates/api_clients/grok_consumer_client/src/error/grok_generic_api_error.rs

@@ -1,82 +0,0 @@
-use cloudflare_errors::cloudflare_error::CloudflareError;
-use std::error::Error;
-use std::fmt::{Display, Formatter};
-use wreq::StatusCode;
-
-#[derive(Debug)]
-pub enum GrokGenericApiError {
-  /// Specific Cloudflare errors.
-  CloudflareError(CloudflareError),
-
-  /// The downloaded index.html did not include the expected data.
-  /// This is necessary for handshakes between client and server.
-  IndexHtmlDidNotIncludeExpectedData {
-    message: String,
-  },
-
-  /// The downloaded javascript did not include the expected data.
-  /// This is necessary for handshakes between client and server.
-  ScriptBodyDidNotIncludeExpectedData {
-    message: String,
-  },
-
-  /// Error creating the websocket due to 401 Unauthorized, likely an authentication error with Grok.
-  /// Unfortunately we can't read the response body text to diagnose more because wreq has some API design issues.
-  LikelyWebsocketAuthentication401,
-
-  /// We received a 403 on websocket upgrade, likely from Cloudflare.
-  /// Unfortunately we can't read the response body text to diagnose more because wreq has some API design issues.
-  LikelyWebsocketCloudflare403,
-
-  /// Unknown status code on websocket upgrade.
-  UnexpectedWebsocketUpgradeStatusCode(StatusCode),
-
-  /// serde_json::Error, likely from JSON deserialization schema mismatch.
-  /// Includes the original body.
-  SerdeResponseParseErrorWithBody(serde_json::Error, String),
-
-  /// serde_json::Error, likely from JSON deserialization schema mismatch.
-  /// Includes the original body.
-  /// Specifically, this is for non-200 (error) responses.
-  SerdeParseErrorWithBodyOnNon200(serde_json::Error, String),
-
-  /// An uncategorized bad HTTP response from Grok.
-  UncategorizedBadResponseWithStatus(StatusCode),
-  
-  /// An uncategorized bad HTTP response from Grok.
-  UncategorizedBadResponseWithStatusAndBody {
-    status_code: StatusCode,
-    body: String,
-  },
-  
-  /// Our previous upload failed
-  UploadFailed,
-
-  /// An error upgrading the connection to a websocket.
-  WreqWebsocketUpgradeError(wreq::Error),
-
-  /// An uncaught error from the API client.
-  WreqError(wreq::Error),
-}
-
-impl Error for GrokGenericApiError {}
-
-impl Display for GrokGenericApiError {
-  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-    match self {
-      Self::CloudflareError(err) => write!(f, "Cloudflare error: {}", err),
-      Self::IndexHtmlDidNotIncludeExpectedData { message } => write!(f, "IndexHtmlDidNotIncludeExpectedData ( message: {} )", message),
-      Self::ScriptBodyDidNotIncludeExpectedData { message } => write!(f, "ScriptBodyDidNotIncludeExpectedData ( message: {} )", message),
-      Self::LikelyWebsocketAuthentication401 => write!(f, "Likely websocket authentication 401 error (probably Grok authentication issue with cookies)"),
-      Self::LikelyWebsocketCloudflare403 => write!(f, "Likely Cloudflare 403 error on websocket upgrade"),
-      Self::UnexpectedWebsocketUpgradeStatusCode(status) => write!(f, "Unexpected websocket upgrade status code: {}", status),
-      Self::SerdeResponseParseErrorWithBody(err, body) => write!(f, "Failed to parse response body: {:?}. Body: {}", err, body),
-      Self::SerdeParseErrorWithBodyOnNon200(err, body) => write!(f, "Failed to parse non-200 response body: {:?}. Body: {}", err, body),
-      Self::UncategorizedBadResponseWithStatus(status) => write!(f, "Uncategorized with status code: {}", status),
-      Self::UncategorizedBadResponseWithStatusAndBody { status_code, body } => write!(f, "Uncategorized bad response: status code {}, body: {}", status_code, body),
-      Self::UploadFailed => write!(f, "Upload failed"),
-      Self::WreqWebsocketUpgradeError(err) => write!(f, "Websocket upgrade error: {}", err),
-      Self::WreqError(err) => write!(f, "Wreq client error: {}", err),
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/error/grok_specific_api_error.rs

@@ -1,78 +0,0 @@
-use std::error::Error;
-use std::fmt::{Display, Formatter};
-
-#[derive(Debug)]
-pub enum GrokSpecificApiError {
-  /// Grok is detecting and blocking our bot, likely due to new secrets use.
-  AutomationBlocked,
-  
-  /// Too many videos are being generated.
-  TooManyVideos,
-  
-
-//  /// Failed to generate a bearer token despite the `200 OK` response.
-//  /// This has been generated by sending old/expired session cookies or empty cookies.
-//  FailedToGenerateBearerToken,
-//
-//  /// The endpoint told us we used an invalid JWT.
-//  InvalidJwt,
-//
-//  /// Sora has responded with "sentinel block".
-//  /// This usually means we need to refresh our sentinel token.
-//  SentinelBlockError,
-//
-//  /// Sora requires users to set up an account username before they can generate images.
-//  /// This requires the user to set up that username.
-//  SoraUsernameNotYetCreated,
-//
-//  /// Sora has responded with "token expired".
-//  /// This (probably?) means we need to refresh our JWT bearer token.
-//  TokenExpiredError,
-//
-//  /// We're sending too many tasks to Sora.
-//  /// Sora is telling us to cool it.
-//  TooManyConcurrentTasks,
-//
-//  // TODO: Try to make concrete error types for these distinct cases.
-//  /// Unauthorized, cookie and/or bearer token expired. We'll need to ask for a refreshed login.
-//  ///
-//  /// Example message, e.g. from the upload endpoint:
-//  ///   {
-//  ///     "error": {
-//  ///       "message": "Your authentication token has expired. Please try signing in again.",
-//  ///       "type": "invalid_request_error",
-//  ///       "param": null,
-//  ///       "code": "token_expired"
-//  ///     }
-//  ///   }
-//  UnauthorizedCookieOrBearerExpired,
-//
-//  /// The Sora sentinel token response is missing fields
-//  SentinelResponseIsMissingFields {
-//    missing_token: bool,
-//    missing_turnstile: bool,
-//    raw_response: String,
-//  },
-}
-
-impl Error for GrokSpecificApiError {}
-
-impl Display for GrokSpecificApiError {
-  fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-    match self {
-      Self::AutomationBlocked => write!(f, "Automation blocked"),
-      Self::TooManyVideos => write!(f, "Too many videos are being generated"),
-
-      //Self::FailedToGenerateBearerToken => write!(f, "Failed to generate bearer token. Probably invalid or expired session cookies."),
-      //Self::InvalidJwt => write!(f, "Invalid JWT token. Please refresh the token."),
-      //Self::SentinelBlockError => write!(f, "Request blocked by Sentinel Token Block. Please refresh the sentinel token."),
-      //Self::SoraUsernameNotYetCreated => write!(f, "Sora username not yet created. Please create a username in the Sora app or website."),
-      //Self::TokenExpiredError => write!(f, "JWT token has expired. Please refresh the token."),
-      //Self::TooManyConcurrentTasks => write!(f, "Too many concurrent tasks. Please wait."),
-      //Self::UnauthorizedCookieOrBearerExpired => write!(f, "Unauthorized: cookie and/or bearer token expired"),
-      //Self::SentinelResponseIsMissingFields { missing_token: token_missing, missing_turnstile: turnstile_missing, raw_response } => {
-      //  write!(f, "Sentinel response is missing fields. token_missing: {}, turnstile_missing: {}. Raw response: {}", token_missing, turnstile_missing, raw_response)
-      //},
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/error/mod.rs

@@ -1,5 +0,0 @@
-pub mod grok_client_error;
-pub mod grok_error;
-pub mod grok_generic_api_error;
-pub mod grok_specific_api_error;
-pub (crate) mod categorize_grok_http_error;

crates/api_clients/grok_consumer_client/src/lib.rs

@@ -1,15 +0,0 @@
-//! A Grok API client.
-
-#[cfg(test)]
-pub (crate) mod test_utils;
-
-// Library utils
-pub (crate) mod client;
-
-// User lib
-pub mod credentials;
-pub mod datatypes;
-pub mod error;
-pub mod recipes;
-pub mod requests;
-pub mod utils;

crates/api_clients/grok_consumer_client/src/recipes/_metadata/video_gen_request_flows.md

@@ -1,93 +0,0 @@
-Upload Image -> Video Flow
-==========================
-
-1. Upload Image
-
-  Endpoint: https://grok.com/rest/app-chat/upload-file
-
-  Response Fields:
-    {
-      **"fileMetadataId": "21a79085-e206-4b0b-88ac-5f2b7a453e45",**
-      **"fileUri": "users/85980643-ffab-4984-a3de-59a608c47d7f/21a79085-e206-4b0b-88ac-5f2b7a453e45/content",**
-      "fileMimeType": "image/jpeg",
-      "fileName": "0_0.jpeg",
-      "parsedFileUri": "",
-      "createTime": "2025-10-21T23:40:41.784448Z",
-      "fileSource": "SELF_UPLOAD_FILE_SOURCE"
-    }
-
-
-  File becomes a 480x640 copy:
-
-  https://assets.grok.com/users/85980643-ffab-4984-a3de-59a608c47d7f/21a79085-e206-4b0b-88ac-5f2b7a453e45/content
-
-2. "Create Media Post" **(needed?)**
-
-  Endpoint: https://grok.com/rest/media/post/create
-
-  Request Fields: 
-    {
-      "mediaType": "MEDIA_POST_TYPE_IMAGE",
-      "mediaUrl": "https://assets.grok.com/users/85980643-ffab-4984-a3de-59a608c47d7f/21a79085-e206-4b0b-88ac-5f2b7a453e45/content"
-    }
-
-  Response: 
-      {
-         "post": {
-            "id": "21a79085-e206-4b0b-88ac-5f2b7a453e45",
-            "userId": "85980643-ffab-4984-a3de-59a608c47d7f",
-            "createTime": "2025-10-21T23:40:41.993980477Z",
-            "prompt": "",
-            "mediaType": "MEDIA_POST_TYPE_IMAGE",
-            "mediaUrl": "https://assets.grok.com/users/85980643-ffab-4984-a3de-59a608c47d7f/21a79085-e206-4b0b-88ac-5f2b7a453e45/content",
-            "mimeType": "image/jpeg",
-            "audioUrls": [],
-            "childPosts": []
-         }
-      }
-
-3. Chat Conversation for Video
-
-  Endpoint: https://grok.com/rest/app-chat/conversations/new
-
-  Request:
-    {
-      "temporary":true,
-      "modelName":"grok-3",
-      "message":"https://assets.grok.com/users/85980643-ffab-4984-a3de-59a608c47d7f/21a79085-e206-4b0b-88ac-5f2b7a453e45/content  --mode=normal",
-      "fileAttachments":["21a79085-e206-4b0b-88ac-5f2b7a453e45"],
-      "toolOverrides": {
-        "videoGen":true
-      }
-    }
-
-Header 'x-xai-request-id: 7cb93340-5fb6-4fb3-b88d-ff9f5d99d456' is just a random UUID and it can be changed.
-
-
-Generate Image -> Video Flow
-============================
-
-(TODO)
-
-2. Create Media Post
-
-  Request
-    {
-      "mediaType": "MEDIA_POST_TYPE_IMAGE",
-      "mediaUrl":"https://imagine-public.x.ai/imagine-public/images/2d8260d7-3b7c-4bc1-84ca-6e82dd892c39.png"
-    }
-
-
-3. Chat Conversation for Video
-
-   Endpoint: https://grok.com/rest/app-chat/conversations/new
-
-   Request:
-     {
-       "temporary": true,
-       "modelName": "grok-3",
-       "message": "https://imagine-public.x.ai/imagine-public/images/2d8260d7-3b7c-4bc1-84ca-6e82dd892c39.png  alien walks down the hallway --mode=custom",
-       "toolOverrides": {
-         "videoGen":true
-       }
-     } 

crates/api_clients/grok_consumer_client/src/recipes/mod.rs

@@ -1,4 +0,0 @@
-pub mod prompt_websocket_image_with_retry;
-pub mod request_client_secrets;
-pub mod upload_image_and_generate_video;
-pub mod upload_image_and_generate_video_with_retry;

crates/api_clients/grok_consumer_client/src/recipes/prompt_websocket_image_with_retry.rs

@@ -1,201 +0,0 @@
-use crate::credentials::grok_cookies::GrokCookies;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::image_websocket::create_listen_websocket::{create_listen_websocket, CreateListenWebsocketArgs};
-use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-use crate::requests::image_websocket::messages::websocket_client_message::{ClientMessageAspectRatio, WebsocketClientMessage};
-use log::{error, info};
-
-pub struct PromptWebsocketImageWithRetryArgs<'a> {
-  pub websocket: &'a mut GrokWebsocket,
-  pub prompt: &'a str,
-  pub aspect_ratio: ClientMessageAspectRatio,
-  pub cookies: &'a GrokCookies,
-}
-
-pub struct PromptWebsocketImageWithRetryResult {
-  pub maybe_new_websocket: Option<GrokWebsocket>,
-}
-
-
-pub async fn prompt_websocket_image_with_retry(args: PromptWebsocketImageWithRetryArgs<'_>) -> Result<PromptWebsocketImageWithRetryResult, GrokError>
-{
-  let mut maybe_new_websocket : Option<GrokWebsocket> = None;
-
-  for i in 0..3 {
-    info!("Starting Grok Websocket image generation (attempt {}) ...", (i+1));
-
-    let message = WebsocketClientMessage::new_image_prompt(args.prompt, args.aspect_ratio);
-
-    let result =
-        if let Some(websocket) = maybe_new_websocket.as_mut() {
-          websocket.send_serializable(&message).await
-        } else {
-          args.websocket.send_serializable(&message).await
-        };
-
-    match result {
-      Ok(()) => {
-        break;
-      }
-      Err(GrokError::Client(GrokClientError::WreqClientError(err))) => {
-        error!("Error sending message to websocket (WreqClientError): {}", err);
-      }
-      Err(GrokError::Client(err)) => {
-        error!("Error sending message to websocket (GrokClientError): {}", err);
-      }
-      Err(GrokError::ApiSpecific(err)) => {
-        error!("Error sending message to websocket (GrokSpecificApiError): {}", err);
-      }
-      Err(GrokError::ApiGeneric(err)) => {
-        error!("Error sending message to websocket (GrokGenericApiError): {}", err);
-      }
-    }
-
-    let result = create_listen_websocket(CreateListenWebsocketArgs {
-      cookies: args.cookies.as_str(),
-    }).await?;
-
-    maybe_new_websocket = Some(GrokWebsocket::new(result));
-  }
-
-  Ok(PromptWebsocketImageWithRetryResult {
-    maybe_new_websocket,
-  })
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::recipes::prompt_websocket_image_with_retry::{prompt_websocket_image_with_retry, PromptWebsocketImageWithRetryArgs};
-  use crate::requests::image_websocket::create_listen_websocket::{create_listen_websocket, CreateListenWebsocketArgs};
-  use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-  use crate::requests::image_websocket::grok_wrapped_websocket::GrokWrappedWebsocket;
-  use crate::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio;
-  use crate::requests::image_websocket::messages::websocket_server_message::WebsocketServerMessage;
-  use crate::requests::image_websocket::prompt_websocket_image::{prompt_websocket_image, PromptWebsocketImageArgs};
-  use crate::test_utils::get_test_cookies::{get_test_cookies, get_typed_test_cookies};
-  use errors::AnyhowResult;
-  use futures::SinkExt;
-  use log::warn;
-  use std::io::Write;
-  use std::time::Duration;
-  use wreq::ws::message::{CloseCode, Message, Utf8Bytes};
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn prompt() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-
-    let cookies = get_typed_test_cookies()?;
-
-    let websocket = create_listen_websocket(CreateListenWebsocketArgs {
-      cookies: cookies.as_str(),
-    }).await?;
-
-    let prompt = "A dinosaur on stilts walking on the beach";
-
-    //let websocket = GrokWrappedWebsocket::new(websocket);
-    let mut websocket = GrokWebsocket::new(websocket);
-
-    //println!("Sending...");
-    //std::io::stdout().flush()?;
-
-    //println!("For our test, we'll close the websocket and retry...");
-    //websocket.websocket
-    //    .close(CloseCode::NORMAL, Utf8Bytes::try_from(Vec::new())?).await?;
-
-    //let mut websocket = GrokWebsocket::new(websocket.websocket);
-
-    let result = prompt_websocket_image_with_retry(PromptWebsocketImageWithRetryArgs {
-      websocket: &mut websocket,
-      prompt,
-      aspect_ratio: ClientMessageAspectRatio::WideThreeByTwo,
-      cookies: &cookies,
-    }).await?;
-
-    println!("Reading...");
-    std::io::stdout().flush()?;
-
-    let mut count = 0;
-
-    println!("Polling.");
-
-    loop {
-      //let maybe_message =
-      //    websocket.get_response_with_timeout(Duration::from_millis(1000)).await?;
-
-      let maybe_message =
-          websocket.try_next_timeout(Duration::from_millis(1000)).await?;
-
-      let mut maybe_raw_text : Option<String> = None;
-
-      let maybe_message = match maybe_message {
-        None => None,
-        Some(Message::Text(text)) => {
-          let maybe_message = WebsocketServerMessage::from_json_str(text.as_str())?;
-          let mut text = text.as_str().to_string();
-          text.truncate(100);
-          //println!("Received message: {}", text);
-          maybe_raw_text = Some(text);
-          Some(maybe_message)
-        },
-        Some(_) => {
-          warn!("Received non-text websocket message.");
-          None
-        },
-      };
-
-      match maybe_message {
-        None => {
-          println!("No message received within timeout.");
-          count = count + 1;
-        }
-        Some(message) => {
-          match message {
-            WebsocketServerMessage::Image(image) => {
-              println!("IMAGE: {:?}", image.percentage_complete);
-              if let Some(percent) = image.percentage_complete {
-                if percent > 90.0 {
-                  println!("Image: {:?}", image);
-                }
-              }
-            }
-            WebsocketServerMessage::Json(json) => {
-              println!("JSON : {:?}", json.percentage_complete);
-              if let Some(percent) = json.percentage_complete {
-                if percent > 90.0 {
-                  println!("Image: {:?}", json);
-                }
-              }
-            }
-            WebsocketServerMessage::Unknown(unknown) => {
-              //let typ = unknown.get("type");
-              let unknown_string = unknown.as_str();
-              println!("[UNKNOWN] websocket message: {:?}", maybe_raw_text);
-            }
-          }
-          count = 0;
-        }
-      }
-
-      // TODO: Make sure we capture all the events for images
-      // TODO: Explore the other APIs
-      // TODO understand the IDs
-      // TODO: Video websocket APIs
-      // TODO: Upload APIs
-
-      if count > 5 {
-        println!("No messages after 5 seconds");
-        break;
-      }
-    }
-
-    println!("Done polling.");
-
-    log::logger().flush();
-
-    assert_eq!(1,2);
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/recipes/request_client_secrets.rs

@@ -1,124 +0,0 @@
-use crate::credentials::grok_client_secrets::GrokClientSecrets;
-use crate::credentials::grok_cookies::GrokCookies;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs};
-use crate::requests::index_page::get_xsid_script::{get_xsid_script, GetXsidScriptArgs};
-use crate::requests::index_page::parsers::index::parse_index_baggage::parse_index_baggage;
-use crate::requests::index_page::parsers::index::parse_index_sentry_trace::parse_index_sentry_trace;
-use crate::requests::index_page::parsers::index::parse_index_svg_paths::parse_svg_paths_from_index_html;
-use crate::requests::index_page::parsers::index::parse_index_user_email::parse_index_user_email;
-use crate::requests::index_page::parsers::index::parse_index_user_id::parse_index_user_id;
-use crate::requests::index_page::parsers::index::parse_index_verification_token::parse_index_verification_token;
-use crate::requests::index_page::parsers::script::parse_script_actions_and_xsid_script_path::parse_script_actions_and_xsid_script_path;
-use crate::requests::index_page::parsers::script::parse_xsid_script_numbers::parse_xsid_script_numbers;
-use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::convert_verification_token_to_loading_anim;
-use crate::requests::index_page::utils::select_svg_path_by_loading_anim::select_svg_path_by_loading_anim;
-
-pub struct RequestClientSecretsArgs<'a> {
-  pub cookies: &'a GrokCookies,
-}
-
-// Video generation needs:
-// 1) baggage -- on index page
-// 2) sentry_trace -- on index page
-// 3) x_statsig
-
-/// Load all the things needed to make requests.
-pub async fn request_client_secrets(args: RequestClientSecretsArgs<'_>) -> Result<GrokClientSecrets, GrokError> {
-
-  let payloads = get_index_page_and_scripts(GetIndexPageAndScriptsArgs {
-    cookie: args.cookies.as_str(),
-  }).await?;
-
-  let baggage = parse_index_baggage(&payloads.index_body_html)
-      .ok_or_else(|| GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData {
-        message: "Index did not include baggage.".to_string()
-      })?;
-
-  let sentry_trace = parse_index_sentry_trace(&payloads.index_body_html)
-      .ok_or_else(|| GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData {
-        message: "Index did not include sentry trace.".to_string()
-      })?;
-
-  let verification_token = parse_index_verification_token(&payloads.index_body_html)
-      .ok_or_else(|| GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData {
-        message: "Index did not include verification token.".to_string()
-      })?;
-
-  let user_id = parse_index_user_id(&payloads.index_body_html)
-      .ok_or_else(|| GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData {
-        message: "Index did not include user id.".to_string()
-      })?;
-
-  // NB: Optional.
-  let maybe_user_email = parse_index_user_email(&payloads.index_body_html);
-
-  // NB: We'll only use one of the several SVG paths
-  let svg_paths = parse_svg_paths_from_index_html(&payloads.index_body_html);
-
-  if svg_paths.is_empty() {
-    return Err(GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData {
-      message: "Index did not include any SVG paths.".to_string(),
-    }.into());
-  }
-
-  let loading_anim = convert_verification_token_to_loading_anim(&verification_token)?;
-
-  let svg_path_data = select_svg_path_by_loading_anim(&svg_paths, &loading_anim)?;
-
-  let actions_and_xsid_script = parse_script_actions_and_xsid_script_path(&payloads.scripts)?;
-
-  let xsid_script = get_xsid_script(GetXsidScriptArgs {
-    client: &payloads.client,
-    html: &payloads.index_body_html,
-    cookie: args.cookies.as_str(),
-    xsid_script_id: &actions_and_xsid_script.xsid_script_path,
-  }).await?;
-
-  let numbers = parse_xsid_script_numbers(&xsid_script.xsid_script_body);
-
-  if numbers.numbers.is_empty() {
-    return Err(GrokGenericApiError::IndexHtmlDidNotIncludeExpectedData {
-      message: "Index did not include any SVG paths.".to_string(),
-    }.into());
-  }
-
-  Ok(GrokClientSecrets {
-    baggage,
-    sentry_trace,
-    verification_token,
-    svg_path_data,
-    numbers,
-    user_id,
-    user_email: maybe_user_email,
-  })
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::recipes::request_client_secrets::{request_client_secrets, RequestClientSecretsArgs};
-  use crate::test_utils::get_test_cookies::get_typed_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookies = get_typed_test_cookies()?;
-
-    let secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &cookies,
-    }).await?;
-
-    println!("Baggage: {:?}", secrets.baggage);
-    println!("Sentry trace: {:?}", secrets.sentry_trace);
-    println!("Verification token: {:?}", secrets.verification_token);
-    println!("SVG path: {:?}", secrets.svg_path_data);
-    println!("Numbers: {:?}", secrets.numbers);
-    println!("User ID: {:?}", secrets.user_id);
-    println!("User Email: {:?}", secrets.user_email);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/recipes/upload_image_and_generate_video.rs

@@ -1,242 +0,0 @@
-use crate::credentials::grok_full_credentials::GrokFullCredentials;
-use crate::datatypes::api::aspect_ratio::AspectRatio;
-use crate::datatypes::api::file_id::FileId;
-use crate::datatypes::api::post_id::PostId;
-use crate::datatypes::api::video_generation_mode::VideoGenerationMode;
-use crate::datatypes::file_upload_spec::FileUploadSpec;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::requests::media_posts::create_media_post::grok_create_media_post::{GrokCreateMediaPost, MediaPostType};
-use crate::requests::media_posts::like_media_post::grok_like_media::GrokLikeMediaPost;
-use crate::requests::upload_file::grok_upload_file::GrokUploadFile;
-use crate::requests::video_chat::grok_video_gen_chat_conversation::{GrokVideoGenChatConversationBuilder, VideoMediaPostType};
-use crate::utils::user_and_file_id_to_image_url::user_and_file_id_to_image_url;
-use crate::utils::user_and_file_id_to_video_url::user_and_file_id_to_video_url;
-use log::{error, info};
-use std::path::Path;
-use std::time::Duration;
-
-pub struct UploadImageAndGenerateVideo<'a, P: AsRef<Path>> {
-  pub full_credentials: &'a GrokFullCredentials,
-
-  // NB: Must be owned.
-  pub file: FileUploadSpec<P>,
-
-  /// Video generation prompt
-  pub prompt: Option<&'a str>,
-
-  /// Aspect ratio for the video.
-  pub aspect_ratio: Option<AspectRatio>,
-
-  /// Mode to generate the video: normal, fun, spicy, etc.
-  pub mode: Option<VideoGenerationMode>,
-
-  /// Wait for the full video to be generated before returning
-  /// By default, the endpoint stays open for 30-ish seconds while
-  /// the video generates and streams back JSON progress updates.
-  /// If we set this to false, we'll wait for the video ID and exit
-  /// the request early, before the video is finished asynchronously
-  /// generating.
-  pub wait_for_generation: bool,
-
-  pub individual_request_timeout: Option<Duration>,
-}
-
-#[derive(Debug, Clone)]
-pub struct ImageUploadAndGenerateVideoResult {
-  /// The ID of the "post"
-  /// This often is equivalent to the upload id / `image_file_id`,
-  /// and this is in the user-facing URL for website navigation to the video.
-  pub post_id: PostId,
-
-  /// The ID of the file we uploaded
-  pub image_file_id: FileId,
-
-  /// The ID of the video we're generating, which may or may not exist yet.
-  pub video_file_id: Option<FileId>,
-
-  /// The URL for the video, which may or may not exist yet.
-  pub video_url: Option<String>,
-
-  /// Whether the video generation has already completed.
-  /// Based on request `wait_for_generation`.
-  pub generation_is_complete: bool,
-}
-
-pub async fn upload_image_and_generate_video<P: AsRef<Path>>(args: UploadImageAndGenerateVideo<'_, P>)
-  -> Result<ImageUploadAndGenerateVideoResult, GrokError>
-{
-
-  info!("Uploading file to Grok...");
-
-  let request = GrokUploadFile {
-    file: args.file,
-    cookie: args.full_credentials.cookies.to_string(),
-    request_timeout: args.individual_request_timeout,
-  };
-
-  let upload_result = request.upload().await?;
-
-  info!("File URI: {:?}", upload_result.file_uri);
-  info!("File Metadata ID: {:?}", upload_result.file_id);
-
-  let upload_file_id = match upload_result.file_id {
-    Some(id) => id,
-    None => {
-      error!("Previous file upload failed. Cannot continue.");
-      return Err(GrokGenericApiError::UploadFailed.into());
-    }
-  };
-
-  let url = user_and_file_id_to_image_url(
-    &args.full_credentials.client_secrets.user_id,
-    &upload_file_id
-  );
-
-  info!("Uploaded URI: {:?}", url);
-
-  info!("Creating media post...");
-
-  let request = GrokCreateMediaPost {
-    user_id: &args.full_credentials.client_secrets.user_id,
-    file_id: &upload_file_id,
-    media_type: MediaPostType::UserUploadedImage,
-    cookie: args.full_credentials.cookies.as_str(),
-    request_timeout: args.individual_request_timeout,
-  };
-
-  let post_result = request.send().await?;
-
-  info!("Posted Image.");
-
-  info!("Video prompt: {:?}", args.prompt);
-  info!("Video aspect ratio: {:?}", args.aspect_ratio);
-
-  info!("Generate video...");
-
-  let request = GrokVideoGenChatConversationBuilder {
-    user_id: &args.full_credentials.client_secrets.user_id,
-    file_id: &upload_file_id,
-    media_type: VideoMediaPostType::UserUploadedImage,
-    cookie: args.full_credentials.cookies.as_str(),
-    prompt: args.prompt,
-    mode: args.mode,
-    aspect_ratio: args.aspect_ratio,
-    request_timeout: args.individual_request_timeout,
-    baggage: &args.full_credentials.client_secrets.baggage,
-    sentry_trace: &args.full_credentials.client_secrets.sentry_trace,
-    verification_token: &args.full_credentials.client_secrets.verification_token,
-    svg_data: &args.full_credentials.client_secrets.svg_path_data,
-    numbers: &args.full_credentials.client_secrets.numbers,
-  };
-
-  let maybe_video_file_id;
-  let generation_is_complete;
-
-  if args.wait_for_generation {
-    info!("Waiting for video generation...");
-    let video_gen_result = request.wait_for_video().await?;
-    maybe_video_file_id = video_gen_result.video_file_id;
-    generation_is_complete = true;
-  } else {
-    info!("Streaming response partially...");
-    let video_gen_result = request.stream_only_video_id().await?;
-    maybe_video_file_id = video_gen_result.video_file_id;
-    generation_is_complete = false;
-  }
-
-  let maybe_video_url = maybe_video_file_id
-      .as_ref()
-      .map(|file_id| {
-        user_and_file_id_to_video_url(&args.full_credentials.client_secrets.user_id, file_id, false)
-      });
-
-  info!("Video Generation Enqueued");
-
-  info!("Liking media...");
-
-  let request = GrokLikeMediaPost {
-    file_id: &upload_file_id,
-    cookie: args.full_credentials.cookies.as_str(),
-    request_timeout: args.individual_request_timeout,
-    baggage: &args.full_credentials.client_secrets.baggage,
-    sentry_trace: &args.full_credentials.client_secrets.sentry_trace,
-    verification_token: &args.full_credentials.client_secrets.verification_token,
-    svg_data: &args.full_credentials.client_secrets.svg_path_data,
-    numbers: &args.full_credentials.client_secrets.numbers,
-  };
-
-  let _like_result = request.send().await?;
-
-  info!("Media Liked");
-
-  Ok(ImageUploadAndGenerateVideoResult {
-    post_id: post_result.post_id,
-    image_file_id: upload_file_id,
-    video_file_id: maybe_video_file_id,
-    video_url: maybe_video_url,
-    generation_is_complete,
-  })
-}
-
-
-#[cfg(test)]
-mod tests {
-  use crate::credentials::grok_full_credentials::GrokFullCredentials;
-  use crate::datatypes::api::aspect_ratio::AspectRatio;
-  use crate::datatypes::file_upload_spec::FileUploadSpec;
-  use crate::recipes::request_client_secrets::{request_client_secrets, RequestClientSecretsArgs};
-  use crate::recipes::upload_image_and_generate_video::{upload_image_and_generate_video, UploadImageAndGenerateVideo};
-  use crate::test_utils::get_test_cookies::get_typed_test_cookies;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-  // Result: GrokUploadFileResponse { file_metadata_id:
-  // Some("acdee48f-9d6f-4bc6-9d06-fcc97dd4418a"), file_uri:
-  // Some("users/85980643-ffab-4984-a3de-59a608c47d7f/acdee48f-9d6f-4bc6-9d06-fcc97dd4418a/content") }
-
-  #[tokio::test]
-  #[ignore] // Client side tests only
-  async fn test_upload_image_and_generate_video() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Info);
-
-    //let image_path = "/Users/bt/Pictures/Zelda 64 Art/FCgYX6tWEAEhpsy.jpg";
-    //let prompt = "our hero link plunges the sword into the pedestal, the temple is glowing with a blue aura";
-
-    let image_path = "/Users/bt/Pictures/Midjourney/hipster_girl.png";
-    let maybe_prompt = Some("an anime girl runs away from a giant t-rex");
-
-    let cookies = get_typed_test_cookies()?;
-
-    let secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &cookies,
-    }).await?;
-
-    println!("[test] Verification Token: {:?}", secrets.verification_token);
-    println!("[test] Sentry Trace: {:?}", secrets.sentry_trace);
-    println!("[test] Numbers: {:?}", secrets.numbers);
-    println!("[test] Svg Path: {:?}", secrets.svg_path_data);
-    println!("[test] Baggage: {:?}", secrets.baggage);
-
-    let credentials = GrokFullCredentials::from_cookies_and_client_secrets(cookies, secrets);
-
-    let video_result = upload_image_and_generate_video(UploadImageAndGenerateVideo {
-      full_credentials: &credentials,
-      file: FileUploadSpec::Path(image_path),
-      prompt: maybe_prompt,
-      aspect_ratio: Some(AspectRatio::Square),
-      mode: None,
-      individual_request_timeout: None,
-      wait_for_generation: false,
-    }).await?;
-
-    println!("[test] Post ID: {:?}", video_result.post_id);
-    println!("[test] Image File ID: {:?}", video_result.image_file_id);
-    println!("[test] Video File ID: {:?}", video_result.video_file_id);
-    println!("[test] Video URL: {:?}", video_result.video_url);
-
-    assert_eq!(1, 2);
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/recipes/upload_image_and_generate_video_with_retry.rs

@@ -1,348 +0,0 @@
-use crate::credentials::grok_client_secrets::GrokClientSecrets;
-use crate::credentials::grok_full_credentials::GrokFullCredentials;
-use crate::datatypes::api::aspect_ratio::AspectRatio;
-use crate::datatypes::api::video_generation_mode::VideoGenerationMode;
-use crate::datatypes::file_upload_spec::FileUploadSpec;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::error::grok_specific_api_error::GrokSpecificApiError;
-use crate::recipes::request_client_secrets::{request_client_secrets, RequestClientSecretsArgs};
-use crate::recipes::upload_image_and_generate_video::{ImageUploadAndGenerateVideoResult, UploadImageAndGenerateVideo};
-use crate::requests::media_posts::create_media_post::grok_create_media_post::{GrokCreateMediaPost, MediaPostType};
-use crate::requests::media_posts::like_media_post::grok_like_media::GrokLikeMediaPost;
-use crate::requests::upload_file::grok_upload_file::GrokUploadFile;
-use crate::requests::video_chat::grok_video_gen_chat_conversation::{GrokVideoGenChatConversationBuilder, VideoMediaPostType};
-use crate::utils::user_and_file_id_to_image_url::user_and_file_id_to_image_url;
-use crate::utils::user_and_file_id_to_video_url::user_and_file_id_to_video_url;
-use log::{error, info};
-use std::path::Path;
-use std::time::Duration;
-
-pub struct UploadImageAndGenerateVideoWithRetry<'a, P: AsRef<Path>> {
-  pub credentials: &'a GrokFullCredentials,
-
-  // NB: Must be owned.
-  pub file: FileUploadSpec<P>,
-
-  /// Video generation prompt
-  pub prompt: Option<&'a str>,
-
-  /// Aspect ratio for the video.
-  pub aspect_ratio: Option<AspectRatio>,
-
-  /// Mode to generate the video: normal, fun, spicy, etc.
-  pub mode: Option<VideoGenerationMode>,
-
-  /// Wait for the full video to be generated before returning
-  /// By default, the endpoint stays open for 30-ish seconds while
-  /// the video generates and streams back JSON progress updates.
-  /// If we set this to false, we'll wait for the video ID and exit
-  /// the request early, before the video is finished asynchronously
-  /// generating.
-  pub wait_for_generation: bool,
-
-  pub individual_request_timeout: Option<Duration>,
-}
-
-#[derive(Clone)]
-pub struct ImageUploadAndGenerateVideoWithRetryResult {
-  /// If new secrets were generated, they're stored here.
-  pub maybe_new_client_secrets: Option<GrokClientSecrets>,
-
-  /// The video generation result.
-  pub upload_result: ImageUploadAndGenerateVideoResult,
-}
-
-pub async fn upload_image_and_generate_video_with_retry<P: AsRef<Path>>(
-  args: UploadImageAndGenerateVideoWithRetry<'_, P>
-) -> Result<ImageUploadAndGenerateVideoWithRetryResult, GrokError> {
-
-  let mut current_full_credentials_ref = args.credentials;
-  let mut maybe_new_credentials = None;
-
-  info!("Uploading file to Grok...");
-
-  let request = GrokUploadFile {
-    file: args.file,
-    cookie: current_full_credentials_ref.cookies.to_string(),
-    request_timeout: args.individual_request_timeout,
-  };
-
-  let upload_result = request.upload().await?;
-
-  info!("File URI: {:?}", upload_result.file_uri);
-  info!("File Metadata ID: {:?}", upload_result.file_id);
-
-  let upload_file_id = match upload_result.file_id {
-    Some(id) => id,
-    None => {
-      error!("Previous file upload failed. Cannot continue.");
-      return Err(GrokGenericApiError::UploadFailed.into());
-    }
-  };
-
-  let url = user_and_file_id_to_image_url(
-    &current_full_credentials_ref.client_secrets.user_id,
-    &upload_file_id
-  );
-
-  info!("Uploaded URI: {:?}", url);
-
-  info!("Creating media post...");
-
-  let request = GrokCreateMediaPost {
-    user_id: &current_full_credentials_ref.client_secrets.user_id,
-    file_id: &upload_file_id,
-    media_type: MediaPostType::UserUploadedImage,
-    cookie: current_full_credentials_ref.cookies.as_str(),
-    request_timeout: args.individual_request_timeout,
-  };
-
-  let post_result = request.send().await?;
-
-  info!("Posted Image.");
-
-  info!("Video prompt: {:?}", args.prompt);
-  info!("Video aspect ratio: {:?}", args.aspect_ratio);
-
-  let mut video_enqueued_successfully = false;
-  let mut generation_is_complete= false;
-  let mut maybe_video_file_id = None;
-  let mut last_error = None;
-
-  for i in 0..3 {
-    info!("Generate video... attempt {}", (i+1));
-
-    let request = GrokVideoGenChatConversationBuilder {
-      user_id: &current_full_credentials_ref.client_secrets.user_id,
-      file_id: &upload_file_id,
-      media_type: VideoMediaPostType::UserUploadedImage,
-      cookie: current_full_credentials_ref.cookies.as_str(),
-      prompt: args.prompt,
-      mode: args.mode,
-      aspect_ratio: args.aspect_ratio,
-      request_timeout: args.individual_request_timeout,
-      baggage: &current_full_credentials_ref.client_secrets.baggage,
-      sentry_trace: &current_full_credentials_ref.client_secrets.sentry_trace,
-      verification_token: &current_full_credentials_ref.client_secrets.verification_token,
-      svg_data: &current_full_credentials_ref.client_secrets.svg_path_data,
-      numbers: &current_full_credentials_ref.client_secrets.numbers,
-    };
-
-    let video_gen_result =
-      if args.wait_for_generation {
-        request.wait_for_video()
-            .await
-            .map(|res| res.video_file_id)
-      } else {
-        request.stream_only_video_id()
-            .await
-            .map(|res| res.video_file_id)
-      };
-
-    match video_gen_result {
-      Ok(res) => {
-        info!("Video Generation Enqueued!");
-        video_enqueued_successfully = true;
-        generation_is_complete = args.wait_for_generation; // If we synchronously waited, it'll be complete.
-        maybe_video_file_id = res;
-        break;
-      }
-      Err(GrokError::ApiSpecific(GrokSpecificApiError::AutomationBlocked)) => {
-        info!("Grok automation blocked; renewing credentials.");
-        last_error = Some(GrokError::ApiSpecific(GrokSpecificApiError::AutomationBlocked));
-      }
-      Err(err) => {
-        info!("Grok error; renewing credentials. Error = {:?}", err);
-        last_error = Some(err);
-      }
-    }
-
-    info!("Refreshing Grok client secrets...");
-
-    let secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &current_full_credentials_ref.cookies,
-    }).await?;
-
-    // NB: This is just to appease the borrow checker. It doesn't like that we're borrowing `maybe_new_credentials`
-    // in a loop and replacing it while borrowed, hence this hack.
-    current_full_credentials_ref = args.credentials;
-
-    maybe_new_credentials = Some(GrokFullCredentials::from_cookies_and_client_secrets(
-      current_full_credentials_ref.cookies.clone(),
-      secrets,
-    ));
-
-    if let Some(creds) = maybe_new_credentials.as_ref() {
-      current_full_credentials_ref = creds;
-    }
-  }
-
-  if !video_enqueued_successfully {
-    if let Some(err) = last_error {
-      return Err(err);
-    } else {
-      // NB: This branch should be impossible as we should always set last_error.
-      return Err(GrokError::Client(GrokClientError::ErrorGeneratingVideo))
-    }
-  }
-
-  let mut video_liked_successfully = false;
-  let mut last_error = None;
-
-  for i in 0..3 {
-    info!("Liking media ... attempt {}", (i+1));
-
-    let request = GrokLikeMediaPost {
-      file_id: &upload_file_id,
-      cookie: current_full_credentials_ref.cookies.as_str(),
-      request_timeout: args.individual_request_timeout,
-      baggage: &current_full_credentials_ref.client_secrets.baggage,
-      sentry_trace: &current_full_credentials_ref.client_secrets.sentry_trace,
-      verification_token: &current_full_credentials_ref.client_secrets.verification_token,
-      svg_data: &current_full_credentials_ref.client_secrets.svg_path_data,
-      numbers: &current_full_credentials_ref.client_secrets.numbers,
-    };
-
-    let like_result = request.send().await;
-
-    match like_result {
-      Ok(_) => {
-        info!("Media Liked");
-        video_liked_successfully = true;
-        break;
-      }
-      Err(GrokError::ApiSpecific(GrokSpecificApiError::AutomationBlocked)) => {
-        info!("Grok automation blocked liking media; renewing credentials.");
-        last_error = Some(GrokError::ApiSpecific(GrokSpecificApiError::AutomationBlocked));
-      }
-      Err(err) => {
-        info!("Grok error liking media; renewing credentials. Error = {:?}", err);
-        last_error = Some(err);
-      }
-    }
-
-    info!("Refreshing Grok client secrets...");
-
-    let secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &current_full_credentials_ref.cookies,
-    }).await?;
-
-    // NB: This is just to appease the borrow checker. It doesn't like that we're borrowing `maybe_new_credentials`
-    // in a loop and replacing it while borrowed, hence this hack.
-    current_full_credentials_ref = args.credentials;
-
-    maybe_new_credentials = Some(GrokFullCredentials::from_cookies_and_client_secrets(
-      current_full_credentials_ref.cookies.clone(),
-      secrets,
-    ));
-
-    if let Some(creds) = maybe_new_credentials.as_ref() {
-      current_full_credentials_ref = creds;
-    }
-  }
-
-  if !video_liked_successfully {
-    if let Some(err) = last_error {
-      return Err(err);
-    } else {
-      // NB: This branch should be impossible as we should always set last_error.
-      return Err(GrokError::Client(GrokClientError::ErrorGeneratingVideo))
-    }
-  }
-
-  let maybe_video_url = maybe_video_file_id
-      .as_ref()
-      .map(|file_id| {
-        user_and_file_id_to_video_url(&current_full_credentials_ref.client_secrets.user_id, file_id, false)
-      });
-
-  Ok(ImageUploadAndGenerateVideoWithRetryResult {
-    maybe_new_client_secrets: maybe_new_credentials
-        .map(|secrets| secrets.client_secrets),
-    upload_result: ImageUploadAndGenerateVideoResult {
-      post_id: post_result.post_id,
-      image_file_id: upload_file_id,
-      video_file_id: maybe_video_file_id,
-      video_url: maybe_video_url,
-      generation_is_complete,
-    }
-  })
-}
-
-
-#[cfg(test)]
-mod tests {
-  use crate::credentials::grok_full_credentials::GrokFullCredentials;
-  use crate::datatypes::api::aspect_ratio::AspectRatio;
-  use crate::datatypes::api::video_generation_mode::VideoGenerationMode;
-  use crate::datatypes::file_upload_spec::FileUploadSpec;
-  use crate::recipes::request_client_secrets::{request_client_secrets, RequestClientSecretsArgs};
-  use crate::recipes::upload_image_and_generate_video_with_retry::{upload_image_and_generate_video_with_retry, UploadImageAndGenerateVideoWithRetry};
-  use crate::test_utils::get_test_cookies::get_typed_test_cookies;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-
-  // Result: GrokUploadFileResponse { file_metadata_id:
-  // Some("acdee48f-9d6f-4bc6-9d06-fcc97dd4418a"), file_uri:
-  // Some("users/85980643-ffab-4984-a3de-59a608c47d7f/acdee48f-9d6f-4bc6-9d06-fcc97dd4418a/content") }
-
-  #[tokio::test]
-  #[ignore] // Client side tests only
-  async fn test_upload_image_and_generate_video() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Info);
-
-    //let image_path = "/Users/bt/Pictures/Zelda 64 Art/FCgYX6tWEAEhpsy.jpg";
-    //let prompt = "our hero link plunges the sword into the pedestal, the temple is glowing with a blue aura";
-
-    let image_path = "/Users/bt/Pictures/Midjourney/hipster_girl.png";
-    let maybe_prompt = Some("an anime girl turns and high fives a giant t-rex");
-
-    let cookies = get_typed_test_cookies()?;
-
-    let mut bad_secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &cookies,
-    }).await?;
-
-    // NB: We're purposely messing these up to trigger "retry".
-    bad_secrets.numbers.numbers = Vec::new();
-    bad_secrets.sentry_trace.0 = "".to_string();
-
-    let credentials = GrokFullCredentials::from_cookies_and_client_secrets(cookies, bad_secrets);
-
-    let result = upload_image_and_generate_video_with_retry(UploadImageAndGenerateVideoWithRetry {
-      credentials: &credentials,
-      file: FileUploadSpec::Path(image_path),
-      prompt: maybe_prompt,
-      mode: Some(VideoGenerationMode::Custom),
-      aspect_ratio: Some(AspectRatio::TallTwoByThree),
-      individual_request_timeout: None,
-      wait_for_generation: false,
-    }).await?;
-
-    let new_secrets = result.maybe_new_client_secrets;
-
-    println!("[test] New secrets ? : {:?}", new_secrets.is_some());
-
-    if let Some(secrets) = new_secrets {
-      println!("[test] Verification Token: {:?}", secrets.verification_token);
-      println!("[test] Sentry Trace: {:?}", secrets.sentry_trace);
-      println!("[test] Numbers: {:?}", secrets.numbers);
-      println!("[test] Svg Path: {:?}", secrets.svg_path_data);
-      println!("[test] Baggage: {:?}", secrets.baggage);
-    }
-
-    let video_result = result.upload_result;
-
-    println!("[test] Post ID: {:?}", video_result.post_id);
-    println!("[test] Image File ID: {:?}", video_result.image_file_id);
-    println!("[test] Video File ID: {:?}", video_result.video_file_id);
-    println!("[test] Video URL: {:?}", video_result.video_url);
-
-    assert_eq!(1, 2);
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/download_video_file/download_video_file.rs

@@ -1,128 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::datatypes::api::file_id::FileId;
-use crate::datatypes::api::user_id::UserId;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::utils::user_and_file_id_to_video_url::user_and_file_id_to_video_url;
-use log::{error, info, warn};
-use std::time::Duration;
-use wreq::header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, CACHE_CONTROL, CONNECTION, CONTENT_TYPE, COOKIE, ORIGIN, REFERER, TE, UPGRADE_INSECURE_REQUESTS, USER_AGENT};
-use wreq::Client;
-use wreq_util::Emulation;
-
-pub struct DownloadVideoFileArgs<'a> {
-  pub cookies: &'a str,
-  pub user_id: &'a UserId,
-  pub file_id: &'a FileId,
-  pub request_timeout: Option<Duration>,
-}
-
-pub struct VideoFile {
-  pub bytes: Vec<u8>,
-}
-
-pub async fn download_video_file(args: DownloadVideoFileArgs<'_>) -> Result<VideoFile, GrokError> {
-
-  let client = Client::builder()
-      .emulation(Emulation::Firefox143)
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  let video_url = user_and_file_id_to_video_url(&args.user_id, &args.file_id, false);
-
-  info!("Video file URL: {}", video_url);
-
-  let mut request_builder = client.get(&video_url)
-      .header(USER_AGENT, FIREFOX_143_MAC_USER_AGENT)
-      .header(ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-      .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
-      .header(ACCEPT_ENCODING, "gzip, deflate, br, zstd")
-      .header(REFERER, "https://grok.com/")
-      .header("Sec-GPC", "1")
-      .header(CONNECTION, "keep-alive")
-      .header(COOKIE, args.cookies)
-      .header(UPGRADE_INSECURE_REQUESTS, "1")
-      .header("sec-fetch-dest", "document")
-      .header("sec-fetch-mode", "navigate")
-      .header("sec-fetch-site", "same-site")
-      .header("sec-fetch-user", "?1")
-      .header("priority", "u=0, i");
-
-
-  if let Some(timeout) = args.request_timeout {
-    request_builder = request_builder.timeout(timeout);
-  }
-
-  let response = request_builder.send()
-      .await
-      .map_err(|err| GrokGenericApiError::WreqError(err))?;
-
-  let status = response.status();
-
-  info!("Download video status: {:?}", status);
-
-  let download_bytes = &response.bytes()
-      .await
-      .map_err(|err| {
-        error!("Error reading Grok create media response body: {:?}", err);
-        GrokGenericApiError::WreqError(err)
-      })?;
-
-  // TODO: Classify errors
-  if !status.is_success() {
-    warn!("Not successful liking media (status: {:?})", status);
-    //  error!("Upload file request returned an error (code {}) : {:?}", status.as_u16(), response_body);
-    //  return Err(classify_general_http_status_code_and_body(status, response_body));
-  }
-
-  Ok(VideoFile {
-    bytes: download_bytes.to_vec(),
-  })
-}
-
-#[cfg(test)]
-mod tests {
-  use std::fs::File;
-  use std::io::Write;
-  use crate::datatypes::api::file_id::FileId;
-  use crate::recipes::request_client_secrets::{request_client_secrets, RequestClientSecretsArgs};
-  use crate::requests::download_video_file::download_video_file::{download_video_file, DownloadVideoFileArgs};
-  use crate::test_utils::get_test_cookies::get_typed_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Client side tests only
-  async fn test_download() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-
-    let cookies = get_typed_test_cookies()?;
-
-    let file_id = FileId("d9b300c6-9562-4e24-a87a-3ede2a53f0bc".to_string()); // Ernest
-
-    let secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &cookies,
-    }).await?;
-
-    println!("Verification Token: {:?}", secrets.verification_token);
-    println!("Sentry Trace: {:?}", secrets.sentry_trace);
-    println!("Numbers: {:?}", secrets.numbers);
-    println!("Svg Path: {:?}", secrets.svg_path_data);
-    println!("Baggage: {:?}", secrets.baggage);
-
-    let download = download_video_file(DownloadVideoFileArgs {
-      cookies: cookies.as_str(),
-      user_id: &secrets.user_id,
-      file_id: &file_id,
-      request_timeout: None,
-    }).await?;
-
-    println!("Video byte length: {:?}", download.bytes.len());
-
-    let mut file = File::create("download_video_file.test.mp4")?;
-    file.write_all(&download.bytes)?;
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/download_video_file/grok_download_video.rs

@@ -1,171 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::datatypes::api::file_id::FileId;
-use crate::datatypes::api::user_id::UserId;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use crate::utils::user_and_file_id_to_video_url::user_and_file_id_to_video_url;
-use futures::{StreamExt, TryStreamExt};
-use log::{error, info, warn};
-use std::io::Write;
-use std::path::Path;
-use std::time::Duration;
-use tokio::fs::File;
-use tokio::io::AsyncWriteExt;
-use wreq::header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, CACHE_CONTROL, CONNECTION, CONTENT_TYPE, COOKIE, ORIGIN, REFERER, TE, UPGRADE_INSECURE_REQUESTS, USER_AGENT};
-use wreq::{Client, Request, Response};
-use wreq_util::Emulation;
-
-pub struct GrokDownloadVideo<'a> {
-  pub cookies: &'a str,
-  pub user_id: &'a UserId,
-  pub file_id: &'a FileId,
-  pub request_timeout: Option<Duration>,
-}
-
-pub struct DownloadedVideoBytes {
-  pub bytes: Vec<u8>,
-}
-
-impl <'a> GrokDownloadVideo<'a> {
-
-  pub async fn download_to_path<P: AsRef<Path>>(&self, path: P) -> Result<(), GrokError> {
-    let response = self.send_request().await?;
-
-    let mut stream = response.bytes_stream();
-
-    let mut file = File::create(&path)
-        .await
-        .map_err(|err| GrokClientError::CannotOpenLocalFileForWriting(err))?;
-
-    //let _r = stream.for_each_concurrent(None, |chunk_result| async {
-    //  match chunk_result {
-    //    Ok(chunk) => {
-    //      file;
-    //      //if let Err(e) = file.write_all(&chunk).await {
-    //      //  //error!("Error writing to file: {}", e);
-    //      //}
-    //    }
-    //    Err(e) => {
-    //      //error!("Error in stream: {}", e);
-    //    }
-    //  }
-    //}).await;
-
-    while let Some(chunk) = stream.try_next()
-        .await
-        .map_err(|err| GrokGenericApiError::WreqError(err))?
-    {
-      file.write_all(&chunk)
-          .await
-          .map_err(|err| GrokClientError::CannotOpenLocalFileForWriting(err))?
-    }
-
-    file.flush().await.unwrap();
-
-    Ok(())
-  }
-
-  pub async fn download_bytes(&self) -> Result<DownloadedVideoBytes, GrokError> {
-    let response = self.send_request().await?;
-
-    let download_bytes = response.bytes()
-        .await
-        .map_err(|err| {
-          error!("Error reading Grok create media response body: {:?}", err);
-          GrokGenericApiError::WreqError(err)
-        })?;
-
-    Ok(DownloadedVideoBytes {
-      bytes: download_bytes.to_vec(),
-    })
-  }
-
-  async fn send_request(&self) -> Result<Response, GrokError> {
-    let client = Client::builder()
-        .emulation(Emulation::Firefox143)
-        .build()
-        .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-    let video_url = user_and_file_id_to_video_url(&self.user_id, &self.file_id, false);
-
-    info!("Video file URL: {}", video_url);
-
-    let mut request_builder = client.get(&video_url)
-        .header(USER_AGENT, FIREFOX_143_MAC_USER_AGENT)
-        .header(ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-        .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
-        .header(ACCEPT_ENCODING, "gzip, deflate, br, zstd")
-        .header(REFERER, "https://grok.com/")
-        .header("Sec-GPC", "1")
-        .header(CONNECTION, "keep-alive")
-        .header(COOKIE, self.cookies.to_string())
-        .header(UPGRADE_INSECURE_REQUESTS, "1")
-        .header("sec-fetch-dest", "document")
-        .header("sec-fetch-mode", "navigate")
-        .header("sec-fetch-site", "same-site")
-        .header("sec-fetch-user", "?1")
-        .header("priority", "u=0, i");
-
-    if let Some(timeout) = self.request_timeout {
-      request_builder = request_builder.timeout(timeout);
-    }
-
-    let response = request_builder.send()
-        .await
-        .map_err(|err| GrokGenericApiError::WreqError(err))?;
-
-    let status = response.status();
-
-    // TODO: Classify errors
-    if !status.is_success() {
-      warn!("Not successful liking media (status: {:?})", status);
-      //  error!("Upload file request returned an error (code {}) : {:?}", status.as_u16(), response_body);
-      //  return Err(classify_general_http_status_code_and_body(status, response_body));
-    }
-
-    Ok(response)
-  }
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::datatypes::api::file_id::FileId;
-  use crate::recipes::request_client_secrets::{request_client_secrets, RequestClientSecretsArgs};
-  use crate::requests::download_video_file::download_video_file::{download_video_file, DownloadVideoFileArgs};
-  use crate::test_utils::get_test_cookies::get_typed_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Client side tests only
-  async fn test_download() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-
-    let cookies = get_typed_test_cookies()?;
-
-    let file_id = FileId("d9b300c6-9562-4e24-a87a-3ede2a53f0bc".to_string()); // Ernest
-
-    let secrets = request_client_secrets(RequestClientSecretsArgs {
-      cookies: &cookies,
-    }).await?;
-
-    println!("Verification Token: {:?}", secrets.verification_token);
-    println!("Sentry Trace: {:?}", secrets.sentry_trace);
-    println!("Numbers: {:?}", secrets.numbers);
-    println!("Svg Path: {:?}", secrets.svg_path_data);
-    println!("Baggage: {:?}", secrets.baggage);
-
-    // TDOO: This implementation is untested.
-    //let download = download_video_file(DownloadVideoFileArgs {
-    //  cookies: cookies.as_str(),
-    //  user_id: &secrets.user_id,
-    //  file_id: &file_id,
-    //  request_timeout: None,
-    //}).await?;
-
-    //println!("Video byte length: {:?}", download.bytes.len());
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/download_video_file/mod.rs

@@ -1,2 +0,0 @@
-pub mod download_video_file;
-pub mod grok_download_video;

crates/api_clients/grok_consumer_client/src/requests/image_websocket/_metadata/tls_fingerprint_checking.rs

@@ -1,81 +0,0 @@
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use std::time::Duration;
-use wreq::header::{HeaderMap, USER_AGENT};
-use wreq::{Client, Proxy};
-use wreq_util::Emulation;
-
-const FINGERPRINT_URL : &str = "https://tools.scrapfly.io/api/tls";
-
-fn build_chrome_140_client() -> Result<Client, GrokError> {
-  Ok(Client::builder()
-      .emulation(Emulation::Chrome140)
-      .connection_verbose(true)
-      .connect_timeout(Duration::from_secs(10))
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?)
-}
-
-fn build_firefox_139_client() -> Result<Client, GrokError> {
-  Ok(Client::builder()
-      .emulation(Emulation::Firefox139) // NB: THIS IS IDENTICAL IN FINGERPRINT TO REAL FIREFOX !!!
-      .connection_verbose(true)
-      .connect_timeout(Duration::from_secs(10))
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?)
-}
-
-fn build_firefox_143_client() -> Result<Client, GrokError> {
-  Ok(Client::builder()
-      .emulation(Emulation::Firefox143) // NB: THIS IS IDENTICAL IN FINGERPRINT TO REAL FIREFOX !!!
-      .connection_verbose(true)
-      .connect_timeout(Duration::from_secs(10))
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?)
-}
-
-fn build_raw_firefox_client_with_proxy() -> Result<Client, GrokError> {
-  let proxy = Proxy::https("http://127.0.0.1:8080")
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-  Ok(Client::builder()
-      .emulation(Emulation::Firefox143)
-      .default_headers(HeaderMap::new())
-      .proxy(proxy)
-      .cert_verification(false)
-      .connection_verbose(true)
-      .connect_timeout(Duration::from_secs(10))
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?)
-}
-
-async fn get_with_client(client: &Client) -> Result<String, GrokError> {
-  let builder = client.get(FINGERPRINT_URL)
-      .header("Accept", "application/json");
-
-  let response = builder.send()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-
-  let body = response.text()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  Ok(body)
-}
-
-#[cfg(test)]
-mod tests {
-  use errors::AnyhowResult;
-  use super::*;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn test() -> AnyhowResult<()> {
-    let client = build_firefox_143_client()?;
-    let body = get_with_client(&client).await?;
-    println!("{}", body);
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/create_listen_websocket.rs

@@ -1,106 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use cloudflare_mitigation::headers::firefox_websocket_http_1_1_headers::get_firefox_websocket_http_1_1_headers;
-use std::ops::{Deref, DerefMut};
-use std::time::Duration;
-use tokio::io::AsyncReadExt;
-use wreq::header::{ACCEPT, ACCEPT_LANGUAGE, CACHE_CONTROL, COOKIE, ORIGIN, PRAGMA, SEC_WEBSOCKET_EXTENSIONS, USER_AGENT};
-use wreq::ws::WebSocket;
-use wreq::Client;
-use wreq_util::Emulation;
-
-const WEBSOCKET_URL: &str = "wss://grok.com/ws/imagine/listen";
-
-pub struct CreateListenWebsocketArgs<'a> {
-  pub cookies: &'a str,
-}
-
-pub async fn create_listen_websocket(args: CreateListenWebsocketArgs<'_>) -> Result<WebSocket, GrokError> {
-  let mut client_builder = Client::builder()
-      .emulation(Emulation::Firefox143)
-      .connection_verbose(true)
-      .connect_timeout(Duration::from_secs(30));
-
-  let client = client_builder
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  // NB: We need to be careful with this because HTTP/1.1 WebSocket handshake is sensitive to header casing and order.
-  // Cloudflare doesn't like lower case header names, and we're sending headers in the right order just to be safe.
-  let builder = client.websocket(WEBSOCKET_URL)
-      .default_headers(false)
-      .orig_headers(get_firefox_websocket_http_1_1_headers())
-      .header(COOKIE, args.cookies)
-      .header(ORIGIN,"https://grok.com")
-      .header(USER_AGENT, FIREFOX_143_MAC_USER_AGENT)
-      .header(SEC_WEBSOCKET_EXTENSIONS, "permessage-deflate")
-      .header(ACCEPT, "*/*")
-      .header(ACCEPT_LANGUAGE, "en-US,en;q=0.5")
-      .header("Sec-Fetch-Dest", "empty")
-      .header("Sec-Fetch-Mode", "websocket")
-      .header("Sec-Fetch-Site", "same-origin")
-      .header(PRAGMA, "no-cache")
-      .header(CACHE_CONTROL, "no-cache");
-
-  let mut response = builder.send()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  let status = response.status();
-
-  match status.as_u16() {
-    101 => {
-      // Successful WebSocket upgrade ... fallthrough.
-    }
-    401 => {
-      // 401 is a likely authentication error with Grok itself.
-      // Unfortunately we can't read the response body text because wreq wants to consume the Response to get it, but the
-      // WebsocketResponse type wraps and doesn't let us move it. It's wreq's fault.
-      return Err(GrokError::ApiGeneric(GrokGenericApiError::LikelyWebsocketAuthentication401));
-    }
-    403 => {
-      // 403 is likely a cloudflare error
-      // Unfortunately we can't read the response body text because wreq wants to consume the Response to get it, but the
-      // WebsocketResponse type wraps and doesn't let us move it. It's wreq's fault.
-      return Err(GrokError::ApiGeneric(GrokGenericApiError::LikelyWebsocketCloudflare403));
-    }
-    _ => {
-      return Err(GrokError::ApiGeneric(GrokGenericApiError::UnexpectedWebsocketUpgradeStatusCode(status)));
-    },
-  }
-
-  let websocket = response.into_websocket()
-      .await
-      .map_err(|err| GrokGenericApiError::WreqWebsocketUpgradeError(err))?;
-
-  Ok(websocket)
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn create() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Trace);
-    //let cookies = SESSION_COOKIES_WITHOUT_CF_CLEARANCE;
-    let cookies = get_test_cookies()?;
-    let args = CreateListenWebsocketArgs {
-      cookies: &cookies,
-    };
-    let result = create_listen_websocket(args).await;
-    if let Err(err) = result {
-      println!("Error: {:?}", err);
-    }
-    log::logger().flush();
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/grok_websocket.rs

@@ -1,108 +0,0 @@
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::image_websocket::messages::websocket_server_message::WebsocketServerMessage;
-use futures::stream::FusedStream;
-use futures::{Stream, TryStreamExt};
-use log::{info, warn};
-use serde::Serialize;
-use std::sync::{Arc, RwLock};
-use std::time::Duration;
-use wreq::ws::message::Message;
-use wreq::ws::WebSocket;
-
-pub struct GrokWebsocket {
-  pub(crate) websocket: WebSocket,
-}
-
-impl GrokWebsocket {
-  pub fn new(websocket: WebSocket) -> Self {
-    Self {
-      websocket,
-    }
-  }
-
-  pub async fn send(&mut self, message: String) -> Result<(), GrokError> {
-    let message = Message::text(message);
-    self.websocket.send(message)
-        .await
-        .map_err(|err| {
-          GrokClientError::WreqClientError(err)
-        })?;
-    Ok(())
-  }
-
-  pub async fn send_serializable<T: Serialize>(&mut self, message: T) -> Result<(), GrokError> {
-    let message_json = serde_json::to_string(&message)
-        .map_err(|err| {
-          warn!("Failed to serialize prompt websocket message: {}", err);
-          GrokClientError::WebsocketRequestSerializationError(err)
-        })?;
-    self.send(message_json).await
-  }
-
-  pub async fn get_response_with_timeout(&mut self, duration: Duration) -> Result<Option<WebsocketServerMessage>, GrokError> {
-    let result = tokio::time::timeout(duration, self.websocket.try_next()).await;
-
-    match result {
-      Err(elapsed) => {
-        info!("Websocket try_next() elapsed without receiving a message: {:?}", elapsed);
-        Ok(None) // Timeout elapsed.
-      }
-      Ok(inner) => {
-        let maybe_message = inner.map_err(GrokClientError::WebsocketReadError)?;
-
-        match maybe_message {
-          None => Ok(None),
-          Some(Message::Text(text)) => {
-            let maybe_message = WebsocketServerMessage::from_json_str(&text)?;
-            Ok(Some(maybe_message))
-          },
-          Some(_) => {
-            warn!("Received non-text websocket message.");
-            Ok(None)
-          },
-        }
-      }
-    }
-  }
-
-  pub async fn try_next_timeout(&mut self, duration: Duration) -> Result<Option<Message>, GrokClientError> {
-    let result = tokio::time::timeout(duration, self.websocket.try_next()).await;
-
-    match result {
-      Err(elapsed) => {
-        info!("Websocket try_next() elapsed without receiving a message: {:?}", elapsed);
-        Ok(None) // Timeout elapsed.
-      }
-      Ok(inner) => {
-        let maybe_message = inner.map_err(GrokClientError::WebsocketReadError)?;
-        Ok(maybe_message)
-      }
-    }
-  }
-
-  // pub async fn is_terminated(&self) -> Result<bool, GrokClientError> {
-  //   match self.websocket.read() {
-  //     Err(_) => Err(GrokClientError::WebsocketLockError.into()),
-  //     Ok(websocket) => Ok(websocket.is_terminated()),
-  //   }
-  // }
-
-  // /// NB: This will block forever if there is no item.
-  // pub async fn try_next(&self) -> Result<Option<Message>, GrokClientError> {
-  //   match self.websocket.write() {
-  //     Err(_) => Err(GrokClientError::WebsocketLockError),
-  //     Ok(mut websocket) => websocket
-  //         .try_next()
-  //         .await
-  //         .map_err(|err| GrokClientError::WebsocketReadError(err)),
-  //   }
-  // }
-
-  // pub fn size_hint(&self) -> Result<(usize, Option<usize>), GrokClientError> {
-  //   match self.websocket.read() {
-  //     Err(_) => Err(GrokClientError::WebsocketLockError),
-  //     Ok(websocket) => Ok(websocket.size_hint())
-  //   }
-  // }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/grok_wrapped_websocket.rs

@@ -1,124 +0,0 @@
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::image_websocket::messages::websocket_server_message::WebsocketServerMessage;
-use futures::stream::FusedStream;
-use futures::{Stream, TryStreamExt};
-use log::{info, warn};
-use serde::Serialize;
-use std::sync::{Arc, RwLock};
-use std::time::Duration;
-use wreq::ws::message::Message;
-use wreq::ws::WebSocket;
-
-#[derive(Clone)]
-pub struct GrokWrappedWebsocket {
-  pub(crate) websocket: Arc<RwLock<WebSocket>>,
-}
-
-impl GrokWrappedWebsocket {
-  pub fn new(websocket: WebSocket) -> Self {
-    Self {
-      websocket: Arc::new(RwLock::new(websocket)),
-    }
-  }
-
-  pub async fn send(&self, message: String) -> Result<(), GrokError> {
-    match self.websocket.write() {
-      Err(err) => Err(GrokClientError::WebsocketLockError.into()),
-      Ok(mut websocket) => {
-        let message = Message::text(message);
-        websocket.send(message)
-            .await
-            .map_err(|err| {
-              GrokClientError::WreqClientError(err)
-            })?;
-        Ok(())
-      },
-    }
-  }
-
-  pub async fn send_serializable<T: Serialize>(&self, message: T) -> Result<(), GrokError> {
-    let message_json = serde_json::to_string(&message)
-        .map_err(|err| {
-          warn!("Failed to serialize prompt websocket message: {}", err);
-          GrokClientError::WebsocketRequestSerializationError(err)
-        })?;
-    self.send(message_json).await
-  }
-
-  pub async fn get_response_with_timeout(&self, duration: Duration) -> Result<Option<WebsocketServerMessage>, GrokError> {
-    match self.websocket.write() {
-      Err(_) => Err(GrokClientError::WebsocketLockError.into()),
-      Ok(mut websocket) => {
-        let result = tokio::time::timeout(duration, websocket.try_next()).await;
-
-        match result {
-          Err(elapsed) => {
-            info!("Websocket try_next() elapsed without receiving a message: {:?}", elapsed);
-            Ok(None) // Timeout elapsed.
-          }
-          Ok(inner) => {
-            let maybe_message = inner.map_err(GrokClientError::WebsocketReadError)?;
-
-            match maybe_message {
-              None => Ok(None),
-              Some(Message::Text(text)) => {
-                let maybe_message = WebsocketServerMessage::from_json_str(&text)?;
-                Ok(Some(maybe_message))
-              },
-              Some(_) => {
-                warn!("Received non-text websocket message.");
-                Ok(None)
-              },
-            }
-          }
-        }
-      },
-    }
-  }
-
-  pub async fn try_next_timeout(&self, duration: Duration) -> Result<Option<Message>, GrokClientError> {
-    match self.websocket.write() {
-      Err(_) => Err(GrokClientError::WebsocketLockError),
-      Ok(mut websocket) => {
-        let result = tokio::time::timeout(duration, websocket.try_next()).await;
-
-        match result {
-          Err(elapsed) => {
-            info!("Websocket try_next() elapsed without receiving a message: {:?}", elapsed);
-            Ok(None) // Timeout elapsed.
-          }
-          Ok(inner) => {
-            let maybe_message = inner.map_err(GrokClientError::WebsocketReadError)?;
-            Ok(maybe_message)
-          }
-        }
-      },
-    }
-  }
-
-  // pub async fn is_terminated(&self) -> Result<bool, GrokClientError> {
-  //   match self.websocket.read() {
-  //     Err(_) => Err(GrokClientError::WebsocketLockError.into()),
-  //     Ok(websocket) => Ok(websocket.is_terminated()),
-  //   }
-  // }
-
-  // /// NB: This will block forever if there is no item.
-  // pub async fn try_next(&self) -> Result<Option<Message>, GrokClientError> {
-  //   match self.websocket.write() {
-  //     Err(_) => Err(GrokClientError::WebsocketLockError),
-  //     Ok(mut websocket) => websocket
-  //         .try_next()
-  //         .await
-  //         .map_err(|err| GrokClientError::WebsocketReadError(err)),
-  //   }
-  // }
-
-  // pub fn size_hint(&self) -> Result<(usize, Option<usize>), GrokClientError> {
-  //   match self.websocket.read() {
-  //     Err(_) => Err(GrokClientError::WebsocketLockError),
-  //     Ok(websocket) => Ok(websocket.size_hint())
-  //   }
-  // }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/listen_for_websocket_images.rs

@@ -1,176 +0,0 @@
-use crate::datatypes::api::request_id::RequestId;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-use crate::requests::image_websocket::grok_wrapped_websocket::GrokWrappedWebsocket;
-use crate::requests::image_websocket::messages::message_image_data::MessageImageData;
-use crate::utils::scrub_blobs_for_debug_logging::scrub_blobs_for_debug_logging;
-use log::info;
-use std::time::{Duration, Instant};
-use wreq::ws::message::Message;
-
-// Number of images we get per prompt by default
-const DEFAULT_IMAGE_COUNT: usize = 6;
-
-pub struct ListenForWebsocketImagesArgs<'a> {
-  pub websocket: &'a mut GrokWebsocket,
-  pub timeout: Duration,
-}
-
-#[derive(Clone, Debug)]
-pub struct ImageResults {
-  pub images: Vec<ImageData>,
-}
-
-#[derive(Clone, Debug)]
-pub struct ImageData {
-  /// The "task" that generated the image
-  /// Multiple images have the same "task"
-  pub request_id: RequestId,
-
-  /// Url where we can download the image.
-  pub url: String,
-
-  /// The user-input prompt
-  pub user_prompt: String,
-
-  /// An X.ai-enriched prompt
-  pub enriched_prompt: String,
-}
-
-pub async fn listen_for_websocket_images(args: ListenForWebsocketImagesArgs<'_>) -> Result<ImageResults, GrokError> {
-  let start = Instant::now();
-  let end_at = start.checked_add(args.timeout);
-  let end_at = match end_at {
-    Some(end_at) => end_at,
-    None => return Err(GrokClientError::TimeoutMathBroken.into()),
-  };
-
-  let mut images = Vec::new();
-
-  loop {
-    if end_at < Instant::now() {
-      break;
-    }
-
-    let maybe_message = args.websocket.try_next_timeout(Duration::from_millis(500)).await?;
-
-    let maybe_message = match maybe_message {
-      None => None,
-      Some(Message::Text(text)) => Some(text.to_string()),
-      Some(_) => {
-        // Got a ping, pong, binary payload, etc.
-        None
-      },
-    };
-
-    let message = match maybe_message {
-      Some(message) => message,
-      None => continue,
-    };
-
-    let maybe_image_data = serde_json::from_str::<MessageImageData>(&message);
-
-    let image_data = match maybe_image_data {
-      Ok(image_data) => image_data,
-      Err(_err) => {
-
-        let message = scrub_blobs_for_debug_logging(&message);
-        println!("Unknown message: {}", message);
-
-        // NB: Might not have been a parse error, since we're parsing irrelevant message payloads too.
-        // This is slightly dangerous to do this way as we might mask true parsing errors with image payloads.
-        continue;
-      }
-    };
-
-    match image_data.percentage_complete {
-      None => continue,
-      Some(percent) => {
-        if percent < 100.0 {
-          continue;
-        }
-      }
-    }
-
-    images.push(ImageData {
-      request_id: RequestId(image_data.request_id),
-      url : image_data.url,
-      enriched_prompt: image_data.full_prompt,
-      user_prompt: image_data.prompt,
-    });
-
-    if images.len() >= DEFAULT_IMAGE_COUNT {
-      info!("{} images generated; we're done polling.", images.len());
-      break;
-    }
-  }
-
-  Ok(ImageResults {
-    images,
-  })
-}
-
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::image_websocket::create_listen_websocket::{create_listen_websocket, CreateListenWebsocketArgs};
-  use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-  use crate::requests::image_websocket::listen_for_websocket_images::{listen_for_websocket_images, ListenForWebsocketImagesArgs};
-  use crate::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio;
-  use crate::requests::image_websocket::prompt_websocket_image::{prompt_websocket_image, PromptWebsocketImageArgs};
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-  use std::io::Write;
-  use std::time::Duration;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn prompt() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Info);
-
-    let prompt = "A tornado hitting the stadium";
-
-    let cookies = get_test_cookies()?;
-
-    let websocket = create_listen_websocket(CreateListenWebsocketArgs {
-      cookies: &cookies,
-    }).await?;
-
-    //let websocket = GrokWrappedWebsocket::new(websocket);
-    let mut websocket = GrokWebsocket::new(websocket);
-
-    println!("Sending...");
-    std::io::stdout().flush()?;
-
-    let _result = prompt_websocket_image(PromptWebsocketImageArgs {
-      websocket: &mut websocket,
-      prompt,
-      aspect_ratio: ClientMessageAspectRatio::WideThreeByTwo,
-    }).await?;
-
-    println!("Reading...");
-    std::io::stdout().flush()?;
-
-    println!("Polling.");
-
-    let images = listen_for_websocket_images(ListenForWebsocketImagesArgs {
-      websocket: &mut websocket,
-      timeout: Duration::from_millis(10_000),
-    }).await?;
-
-    println!("Done polling. Images: {}", images.images.len());
-
-    for image in images.images {
-      println!("Image: {:?}", image);
-    }
-
-    log::logger().flush();
-
-    assert_eq!(1,2);
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/listen_for_websocket_request_id.rs

@@ -1,127 +0,0 @@
-use crate::datatypes::api::request_id::RequestId;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-use crate::requests::image_websocket::messages::message_request_id::MessageRequestId;
-use std::time::{Duration, Instant};
-use wreq::ws::message::Message;
-
-pub struct ListenForWebsocketRequestIdArgs<'a> {
-  pub websocket: &'a mut GrokWebsocket,
-  pub timeout: Duration,
-}
-
-#[derive(Clone, Debug)]
-pub struct RequestIdResult {
-  pub request_id: Option<RequestId>,
-}
-
-pub async fn listen_for_websocket_request_id(args: ListenForWebsocketRequestIdArgs<'_>) -> Result<RequestIdResult, GrokError> {
-  let start = Instant::now();
-  let end_at = start.checked_add(args.timeout);
-  let end_at = match end_at {
-    Some(end_at) => end_at,
-    None => return Err(GrokClientError::TimeoutMathBroken.into()),
-  };
-
-  let mut request_id = None;
-
-  loop {
-    if end_at < Instant::now() {
-      break;
-    }
-
-    let maybe_message = args.websocket.try_next_timeout(Duration::from_millis(500)).await?;
-
-    let maybe_message = match maybe_message {
-      None => None,
-      Some(Message::Text(text)) => Some(text.to_string()),
-      Some(_) => {
-        // Got a ping, pong, binary payload, etc.
-        None
-      },
-    };
-
-    let message = match maybe_message {
-      Some(message) => message,
-      None => continue,
-    };
-
-    let maybe_parsed = serde_json::from_str::<MessageRequestId>(&message);
-
-    let parsed = match maybe_parsed {
-      Ok(parsed) => parsed,
-      Err(_err) => {
-        // NB: Might not have been a parse error, since we're parsing irrelevant message payloads too.
-        // This is slightly dangerous to do this way as we might mask true parsing errors with image payloads.
-        continue;
-      }
-    };
-
-    request_id = Some(RequestId(parsed.request_id));
-    break;
-  }
-
-  Ok(RequestIdResult {
-    request_id,
-  })
-}
-
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::image_websocket::create_listen_websocket::{create_listen_websocket, CreateListenWebsocketArgs};
-  use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-  use crate::requests::image_websocket::listen_for_websocket_request_id::{listen_for_websocket_request_id, ListenForWebsocketRequestIdArgs};
-  use crate::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio;
-  use crate::requests::image_websocket::prompt_websocket_image::{prompt_websocket_image, PromptWebsocketImageArgs};
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-  use std::io::Write;
-  use std::time::Duration;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn prompt() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Info);
-
-    let prompt = "A tiny tornado on a desk";
-
-    let cookies = get_test_cookies()?;
-
-    let mut websocket = create_listen_websocket(CreateListenWebsocketArgs {
-      cookies: &cookies,
-    }).await?;
-
-    //let websocket = GrokWrappedWebsocket::new(websocket);
-    let mut websocket = GrokWebsocket::new(websocket);
-
-    println!("Sending...");
-    std::io::stdout().flush()?;
-
-    let _result = prompt_websocket_image(PromptWebsocketImageArgs {
-      websocket: &mut websocket,
-      prompt,
-      aspect_ratio: ClientMessageAspectRatio::Square,
-    }).await?;
-
-    println!("Reading...");
-    std::io::stdout().flush()?;
-
-    println!("Polling.");
-
-    let request_data = listen_for_websocket_request_id(ListenForWebsocketRequestIdArgs {
-      websocket: &mut websocket,
-      timeout: Duration::from_millis(10_000),
-    }).await?;
-
-    println!("Done polling. Request Data: {:?}", request_data);
-    log::logger().flush();
-
-    assert_eq!(1,2);
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/messages/message_image_data.rs

@@ -1,63 +0,0 @@
-use serde::Deserialize;
-
-
-/// Images with a binary blob and URLs.
-/// We may receive several of these for a single prompt.
-#[derive(Deserialize, Clone, Debug)]
-pub struct MessageImageData {
-  /// UUID.
-  /// Probably an identifier for the original request/prompt.
-  pub request_id: String, // Option<String>,
-
-  /// UUID.
-  /// This turns out to be the file ID in the image URL (and `job_id`).
-  pub id: Option<String>,
-
-  /// UUID.
-  /// This turns out to be the file ID in the image URL (and `id`).
-  pub job_id: Option<String>,
-
-  /// Eg. null, "0", "50", "100"
-  pub percentage_complete: Option<f32>,
-
-  /// The user's prompt
-  pub prompt: String,  //pub prompt: Option<String>,
-
-  /// The X.ai enriched prompt
-  pub full_prompt: String, //pub full_prompt: Option<String>,
-
-  /// URL to the image.
-  pub url: String, //pub url: Option<String>,
-  
-  // We don't need to decode - let's just download again
-  // /// Base64 encoded image blob.
-  // pub blob: Option<String>,
-
-  // /// NSFW flag
-  // pub r_rated: Option<bool>,
-
-  // /// Name of the model used to generate the image
-  // /// eg. "imagine_h_1", "imagine_x_1"
-  // pub model_name: Option<String>,
-}
-
-/*
-Received websocket message: {
-  "type": "image",
-  "blob": "...",
-  "job_id": "260270db-2515-470b-a730-1fe87e11d21c",
-  "grid_index": 0,
-  "is_alteration": false,
-  "order": 0,
-  "section_id": 0,
-  "request_id": "57b94c87-ae13-4ade-8fc0-ebd39fb511ec",
-  "url": "https://imagine-public.x.ai/imagine-public/images/260270db-2515-470b-a730-1fe87e11d21c.png",
-  "id": "260270db-2515-470b-a730-1fe87e11d21c",
-  "prompt": "a dog riding a motorcycle",
-  "full_prompt": "A dog sits in a motorcycle sidecar on a paved road with a natural background.",
-  "percentage_complete": 100.0,
-  "model_name": "imagine_x_1",
-  "suggested_moderation_rule": "MODERATION_RULE_DEFAULT",
-  "r_rated":false
-}
-*/

crates/api_clients/grok_consumer_client/src/requests/image_websocket/messages/message_request_id.rs

@@ -1,11 +0,0 @@
-use serde::Deserialize;
-
-/// This could intercept any message with a "request_id"
-/// We're going to try to use these to end the image prompting early.
-/// If we're not careful, we might accidentally ingest completed image messages this way.
-#[derive(Deserialize, Clone, Debug)]
-pub struct MessageRequestId {
-  /// UUID.
-  /// Probably an identifier for the original request/prompt.
-  pub request_id: String, // Option<String>,
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/messages/mod.rs

@@ -1,4 +0,0 @@
-pub mod message_image_data;
-pub mod message_request_id;
-pub mod websocket_client_message;
-pub mod websocket_server_message;

crates/api_clients/grok_consumer_client/src/requests/image_websocket/messages/websocket_client_message.rs

@@ -1,90 +0,0 @@
-use chrono::Utc;
-use serde::Serialize;
-use uuid::Uuid;
-
-#[derive(Serialize)]
-pub struct WebsocketClientMessage {
-  /// Type of message
-  /// eg. "conversation.item.create"
-  pub r#type: String,
-
-  /// Unix timestamp
-  pub timestamp: u64,
-
-  pub item: ClientMessageItem,
-}
-
-#[derive(Serialize)]
-pub struct ClientMessageItem {
-  /// Type of item
-  /// eg. "message"
-  pub r#type: String,
-
-  pub content: Vec<ClientMessageItemContent>,
-}
-
-#[derive(Serialize)]
-pub struct ClientMessageItemContent {
-  /// UUID
-  #[serde(rename="requestId")]
-  pub request_id: String,
-
-  /// Type of item content
-  /// eg. "input_text"
-  pub r#type: String,
-
-  /// The prompt for the request
-  pub text: String,
-
-  pub properties: ClientMessageItemContentProperties,
-}
-
-#[derive(Serialize)]
-pub struct ClientMessageItemContentProperties {
-  pub section_count: usize,
-  pub is_kids_mode: bool,
-  pub enable_nsfw: bool,
-  pub skip_upsampler: bool,
-  pub is_initial: bool,
-  pub aspect_ratio: ClientMessageAspectRatio,
-}
-
-#[derive(Serialize, Clone, Copy, Debug)]
-pub enum ClientMessageAspectRatio {
-  #[serde(rename = "2:3")]
-  TallTwoByThree,
-
-  #[serde(rename = "3:2")]
-  WideThreeByTwo,
-
-  #[serde(rename = "1:1")]
-  Square,
-}
-
-impl WebsocketClientMessage {
-  /// Create a new image prompt websocket client message
-  pub fn new_image_prompt(prompt: &str, aspect_ratio: ClientMessageAspectRatio) -> Self {
-    Self {
-      r#type: "conversation.item.create".to_string(),
-      timestamp: Utc::now().timestamp_millis() as u64,
-      item: ClientMessageItem {
-        r#type: "message".to_string(),
-        content: vec![
-          ClientMessageItemContent {
-            request_id: Uuid::new_v4().to_string(),
-            r#type: "input_text".to_string(),
-            text: prompt.to_string(),
-            properties: ClientMessageItemContentProperties {
-              section_count: 0,
-              is_kids_mode: false,
-              enable_nsfw: true,
-              skip_upsampler: false,
-              is_initial: false,
-              aspect_ratio,
-            },
-          }
-        ],
-      },
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/messages/websocket_server_message.rs

@@ -1,205 +0,0 @@
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use serde::Deserialize;
-
-#[derive(Deserialize, Clone, Debug)]
-#[serde(tag = "type")]
-pub enum WebsocketServerMessage {
-  /// Json with root `type = "image"`
-  #[serde(rename = "image")]
-  Image(ImageDataMessage),
-
-  /// Json with root `type = "json"`
-  #[serde(rename = "json")]
-  Json(JsonDataMessage),
-
-  /// Captures any other JSON messages.
-  #[serde(untagged)]
-  Unknown(serde_json::Value),
-  //Unknown(String),
-}
-
-/// Images with a binary blob and URLs.
-/// We may receive several of these for a single prompt.
-#[derive(Deserialize, Clone, Debug)]
-pub struct ImageDataMessage {
-  /// UUID.
-  pub id: Option<String>,
-
-  /// UUID.
-  pub job_id: Option<String>,
-
-  /// UUID.
-  pub request_id: Option<String>,
-
-  /// Eg. "0", "50", "100"
-  pub percentage_complete: Option<f32>,
-
-  pub prompt: Option<String>,
-  
-  /// The enriched prompt
-  pub full_prompt: Option<String>,
-
-  /// Base64 encoded image blob.
-  pub blob: Option<String>,
-
-  /// URL to the image.
-  pub url: Option<String>,
-
-  /// NSFW flag
-  pub r_rated: Option<bool>,
-
-  /// Name of the model used to generate the image
-  /// eg. "imagine_h_1"
-  pub model_name: Option<String>,
-}
-
-#[derive(Deserialize, Clone, Debug)]
-pub struct JsonDataMessage {
-  /// UUID.
-  pub job_id: Option<String>,
-
-  /// UUID.
-  pub request_id: Option<String>,
-
-  /// Eg. "0", "50", "100"
-  pub percentage_complete: Option<f32>,
-
-  pub prompt: Option<String>,
-  pub full_prompt: Option<String>,
-
-  /// NSFW flag
-  pub r_rated: Option<bool>,
-
-  /// UUID.
-  /// NB: This is observed to be nullable.
-  pub image_id: Option<String>,
-}
-
-impl WebsocketServerMessage {
-  pub fn from_json_str(json_str: &str) -> Result<Self, GrokGenericApiError> {
-    Ok(serde_json::from_str(json_str)
-        .map_err(|err| GrokGenericApiError::SerdeResponseParseErrorWithBody(err, json_str.to_string()))?)
-  }
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::image_websocket::messages::websocket_server_message::WebsocketServerMessage;
-
-  fn json_path(file_name: &str) -> String {
-    // NB: Cargo runs tests with the crate root as the working directory.
-    format!("test_data/websocket_messages/{}", file_name)
-  }
-
-  #[test]
-  fn test_image_complete() -> anyhow::Result<()> {
-    let path = json_path("create_image_complete_response.json");
-    let data = std::fs::read_to_string(path)?;
-    //let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-    match message {
-      WebsocketServerMessage::Image(image_data) => {
-        assert_eq!(image_data.percentage_complete, Some(50.0));
-        assert!(image_data.url.is_some());
-      },
-      _ => panic!("Expected ImageData message"),
-    }
-
-    Ok(())
-  }
-
-  #[test]
-  fn test_image_status_complete() -> anyhow::Result<()> {
-    let path = json_path("create_image_complete_status_notification.json");
-    let data = std::fs::read_to_string(path)?;
-    //let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-    match message {
-      WebsocketServerMessage::Json(image_data) => {
-        assert_eq!(image_data.percentage_complete, Some(100.0));
-      },
-      _ => panic!("Expected JsonData message"),
-    }
-
-    Ok(())
-  }
-
-  #[test]
-  fn test_in_progress() -> anyhow::Result<()> {
-    let path = json_path("create_image_in_progress_response.json");
-    let data = std::fs::read_to_string(path)?;
-    //let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-    match message {
-      WebsocketServerMessage::Json(image_data) => {
-        assert_eq!(image_data.percentage_complete, Some(0.0));
-      },
-      _ => panic!("Expected JsonData message"),
-    }
-
-    Ok(())
-  }
-
-  #[test]
-  fn test_wire() -> anyhow::Result<()> {
-    let path = json_path("wire.json");
-    let data = std::fs::read_to_string(path)?;
-    //let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-    match message {
-      WebsocketServerMessage::Json(_value)=> {},
-      _ => panic!("Expected Json message"),
-    }
-
-    Ok(())
-  }
-
-  #[test]
-  fn test_wire_formatted() -> anyhow::Result<()> {
-    let path = json_path("wire_formatted.json");
-    let data = std::fs::read_to_string(path)?;
-    //let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-    match message {
-      WebsocketServerMessage::Json(_value)=> {},
-      _ => panic!("Expected Json message"),
-    }
-
-    Ok(())
-  }
-
-  #[test]
-  fn test_empty() -> anyhow::Result<()> {
-    let path = json_path("empty.json");
-    let data = std::fs::read_to_string(path)?;
-    //let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-     match message {
-      WebsocketServerMessage::Unknown(_value)=> {},
-      _ => panic!("Expected Unknown message"),
-    }
-
-    Ok(())
-  }
-
-  #[test]
-  fn test_other_type() -> anyhow::Result<()> {
-    let path = json_path("other_type.json");
-    let data = std::fs::read_to_string(path)?;
-    ///let message: WebsocketServerMessage = serde_json::from_str(&data)?;
-    let message = WebsocketServerMessage::from_json_str(&data)?;
-
-    match message {
-      WebsocketServerMessage::Unknown(_value)=> {},
-      _ => panic!("Expected Unknown message"),
-    }
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/image_websocket/mod.rs

@@ -1,7 +0,0 @@
-pub mod grok_wrapped_websocket;
-pub mod create_listen_websocket;
-pub mod listen_for_websocket_images;
-pub mod listen_for_websocket_request_id;
-pub mod messages;
-pub mod prompt_websocket_image;
-pub mod grok_websocket;

crates/api_clients/grok_consumer_client/src/requests/image_websocket/prompt_websocket_image.rs

@@ -1,171 +0,0 @@
-use crate::error::grok_error::GrokError;
-use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-use crate::requests::image_websocket::messages::websocket_client_message::{ClientMessageAspectRatio, WebsocketClientMessage};
-
-pub struct PromptWebsocketImageArgs<'a> {
-  pub websocket: &'a mut GrokWebsocket,
-  pub prompt: &'a str,
-  pub aspect_ratio: ClientMessageAspectRatio,
-}
-
-
-pub async fn prompt_websocket_image(args: PromptWebsocketImageArgs<'_>) -> Result<(), GrokError> {
-  let message = WebsocketClientMessage::new_image_prompt(args.prompt, args.aspect_ratio);
-
-  args.websocket.send_serializable(&message).await?;
-
-  Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::image_websocket::create_listen_websocket::{create_listen_websocket, CreateListenWebsocketArgs};
-  use crate::requests::image_websocket::grok_websocket::GrokWebsocket;
-  use crate::requests::image_websocket::grok_wrapped_websocket::GrokWrappedWebsocket;
-  use crate::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio;
-  use crate::requests::image_websocket::messages::websocket_server_message::WebsocketServerMessage;
-  use crate::requests::image_websocket::prompt_websocket_image::{prompt_websocket_image, PromptWebsocketImageArgs};
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-  use log::warn;
-  use std::io::Write;
-  use std::time::Duration;
-  use wreq::ws::message::Message;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn prompt() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-
-    let cookies = get_test_cookies()?;
-
-    let websocket = create_listen_websocket(CreateListenWebsocketArgs {
-      cookies: &cookies,
-    }).await?;
-
-    //let websocket = GrokWrappedWebsocket::new(websocket);
-    let mut websocket = GrokWebsocket::new(websocket);
-
-    println!("Sending...");
-    std::io::stdout().flush()?;
-
-    let result = prompt_websocket_image(PromptWebsocketImageArgs {
-      websocket: &mut websocket,
-      prompt: "a dog riding a motorcycle",
-      aspect_ratio: ClientMessageAspectRatio::WideThreeByTwo,
-    }).await?;
-
-    println!("Reading...");
-    std::io::stdout().flush()?;
-
-    let mut count = 0;
-
-    println!("Polling.");
-
-    //while let Some(message) = websocket.try_next().await? {
-    //  println!("[{count}] Received websocket message: {:?}", message);
-    //  count = count + 1;
-    //  tokio::time::sleep(Duration::from_millis(1000)).await;
-    //  if count > 1000 {
-    //    break;
-    //  }
-    //}
-
-    //loop {
-    //  count = count + 1;
-    //  let bounds = websocket.size_hint()?;
-    //  println!("Websocket size hint: {:?}", bounds);
-    //  tokio::time::sleep(Duration::from_millis(1000)).await;
-    //  if count > 30 {
-    //    break;
-    //  }
-    //}
-
-    //loop {
-    //  let maybe_message =
-    //      websocket.try_next_timeout(Duration::from_millis(1000)).await?;
-    //  match maybe_message {
-    //    None => {
-    //      println!("No message received within timeout.");
-    //      count = count + 1;
-    //    }
-    //    Some(message) => {
-    //      println!("[{count}] Received websocket message!");//: {:?}", message);
-    //      count = 0;
-    //    }
-    //  }
-    //  if count > 5 {
-    //    println!("No messages after 5 seconds");
-    //    break;
-    //  }
-    //}
-
-    loop {
-      //let maybe_message =
-      //    websocket.get_response_with_timeout(Duration::from_millis(1000)).await?;
-
-      let maybe_message =
-          websocket.try_next_timeout(Duration::from_millis(1000)).await?;
-
-      let mut maybe_raw_text : Option<String> = None;
-
-      let maybe_message = match maybe_message {
-        None => None,
-        Some(Message::Text(text)) => {
-          let maybe_message = WebsocketServerMessage::from_json_str(text.as_str())?;
-          let mut text = text.as_str().to_string();
-          text.truncate(100);
-          //println!("Received message: {}", text);
-          maybe_raw_text = Some(text);
-          Some(maybe_message)
-        },
-        Some(_) => {
-          warn!("Received non-text websocket message.");
-          None
-        },
-      };
-
-      match maybe_message {
-        None => {
-          println!("No message received within timeout.");
-          count = count + 1;
-        }
-        Some(message) => {
-          match message {
-            WebsocketServerMessage::Image(image) => {
-              println!("IMAGE: {:?}", image.percentage_complete);
-            }
-            WebsocketServerMessage::Json(json) => {
-              println!("JSON : {:?}", json.percentage_complete);
-            }
-            WebsocketServerMessage::Unknown(unknown) => {
-              //let typ = unknown.get("type");
-              let unknown_string = unknown.as_str();
-              println!("[UNKNOWN] websocket message: {:?}", maybe_raw_text);
-            }
-          }
-          count = 0;
-        }
-      }
-
-      // TODO: Make sure we capture all the events for images
-      // TODO: Explore the other APIs
-      // TODO understand the IDs
-      // TODO: Video websocket APIs
-      // TODO: Upload APIs
-
-      if count > 5 {
-        println!("No messages after 5 seconds");
-        break;
-      }
-    }
-
-    println!("Done polling.");
-
-    log::logger().flush();
-
-    assert_eq!(1,2);
-
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/get_index_page.rs

@@ -1,102 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::error::grok_generic_api_error::GrokGenericApiError;
-use log::info;
-use std::ops::Deref;
-use wreq::header::{ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, CACHE_CONTROL, CONNECTION, CONTENT_TYPE, COOKIE, ORIGIN, PRAGMA, REFERER, TE, UPGRADE, USER_AGENT};
-use wreq::ws::message::Message;
-use wreq::Client;
-use wreq_util::Emulation;
-
-const INDEX_URL: &str = "https://grok.com";
-
-pub struct GetIndexPageArgs<'a> {
-  pub cookie: &'a str,
-}
-
-pub struct IndexPage {
-  pub body: String,
-}
-
-pub async fn get_index(args: GetIndexPageArgs<'_>) -> Result<IndexPage, GrokError> {
-  info!("Building client...");
-
-  let client = Client::builder()
-      .emulation(Emulation::Firefox143)
-      .build()
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  info!("Configuring client...");
-
-  let builder = client.get(INDEX_URL)
-      .header("User-Agent", FIREFOX_143_MAC_USER_AGENT)
-      .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-      .header("Accept-Encoding", "gzip, deflate, br, zstd")
-      .header("Accept-Language", "en-US,en;q=0.5")
-      .header("Connection", "keep-alive")
-      .header("Sec-Fetch-Dest", "document")
-      .header("Sec-Fetch-Mode", "navigate")
-      .header("Sec-Fetch-Site", "none")
-      .header("Sec-Fetch-User", "?1")
-      .header("Sec-GPC", "1")
-      .header("priority", "u=0, i")
-      .header("Pragma", "no-cache")
-      .header("Cache-Control", "no-cache")
-      .header("Cookie", args.cookie.to_string())
-      .header("TE", "trailers");
-
-  info!("Sending request for index page...");
-
-  let response = builder.send()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  let status = response.status();
-  println!("Status: {}", status);
-
-  //let body = response.text()
-  //    .await
-  //    .map_err(|err| GrokGenericApiError::WreqError(err))?;
-
-  // stblid=08552693-0377-49d3-b17f-8e4c68b153ec
-  for cookie in response.cookies() {
-    println!("Cookie: {}={}", cookie.name(), cookie.value());
-  }
-
-  for (name, value) in response.headers() {
-    println!("Header: {}: {}", name.as_str(), value.to_str().unwrap());
-  }
-
-  let body = response.text()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  Ok(IndexPage {
-    body,
-  })
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use crate::test_utils::setup_test_logging::setup_test_logging;
-  use errors::AnyhowResult;
-  use log::LevelFilter;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn test() -> AnyhowResult<()> {
-    setup_test_logging(LevelFilter::Trace);
-    let cookie = get_test_cookies()?;
-    let args = GetIndexPageArgs {
-      cookie: &cookie,
-    };
-    let result = get_index(args).await?;
-    println!("Body:\n\n");
-    println!("{}", result.body);
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/get_index_page_and_scripts.rs

@@ -1,88 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::index_page::parsers::index::parse_index_script_list::parse_index_svg_paths;
-use crate::requests::index_page::requests::get_index_page_scripts_with_client::{get_index_page_scripts_with_client, GetIndexPageScriptsArgs};
-use crate::requests::index_page::requests::get_index_page_with_client::{get_index_page_with_client, GetIndexPageWithClientArgs};
-use crate::utils::create_firefox_client::create_firefox_client;
-use log::info;
-use std::collections::HashMap;
-use wreq::Client;
-use wreq_util::Emulation;
-
-const INDEX_URL: &str = "https://grok.com";
-
-pub struct GetIndexPageAndScriptsArgs<'a> {
-  pub cookie: &'a str,
-}
-
-pub struct IndexPageAndScripts {
-  /// Index.html
-  pub index_body_html: String,
-
-  /// Script path -> Javascript payload.
-  pub scripts: HashMap<String, String>,
-  
-  // The client that we used.
-  pub client: Client,
-}
-
-pub async fn get_index_page_and_scripts(args: GetIndexPageAndScriptsArgs<'_>) -> Result<IndexPageAndScripts, GrokError> {
-  info!("Building client...");
-  let client = create_firefox_client()?;
-
-  info!("Fetching index page...");
-
-  let index = get_index_page_with_client(GetIndexPageWithClientArgs {
-    client: &client,
-    cookie: args.cookie,
-  }).await?;
-
-  let scripts = parse_index_svg_paths(&index.body);
-
-  info!("Fetching {} scripts...", scripts.len());
-
-  let scripts = get_index_page_scripts_with_client(GetIndexPageScriptsArgs {
-    client: &client,
-    cookie: &args.cookie,
-    scripts: &scripts,
-  }).await?;
-
-  Ok(IndexPageAndScripts {
-    index_body_html: index.body,
-    scripts,
-    client,
-  })
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs};
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn test() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-    let cookie = get_test_cookies()?;
-    let result = get_index_page_and_scripts(GetIndexPageAndScriptsArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let mut truncated_body = result.index_body_html.clone();
-    truncated_body.truncate(500);
-
-    println!("Body:\n\n");
-    println!("{}", truncated_body);
-
-    for (k, v) in result.scripts.iter() {
-      let mut body = v.to_string();
-      body.truncate(100);
-      println!("{}: {}\n\n", k, body);
-    }
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/get_numbers.rs

@@ -1,87 +0,0 @@
-use crate::datatypes::api::xsid_numbers::XsidNumbers;
-use crate::error::grok_error::GrokError;
-use crate::requests::index_page::parsers::index::parse_index_on_demand_script::parse_index_on_demand_script;
-use crate::requests::index_page::parsers::script::parse_xsid_script_numbers::parse_xsid_script_numbers;
-use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs};
-use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::LoadingAnim;
-use wreq::Client;
-// self.svg_data, self.numbers = Parser.parse_values(c_request.text, self.anim, self.xsid_script)
-
-pub struct GetNumbersArgs<'a> {
-  pub client: &'a Client,
-  pub html: &'a str,
-  pub cookie: &'a str,
-  pub loading_anim: &'a LoadingAnim,
-  pub xsid_script_id: &'a str,
-}
-
-pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result<XsidNumbers, GrokError> {
-
-  let script_link = match args.xsid_script_id {
-    "ondemand.s" => {
-      // TODO: Not sure this is relevant anymore. Not a full or correct implementation of this branch.
-      let on_demand = parse_index_on_demand_script(&args.html);
-      let on_demand = on_demand.unwrap_or_else(|| "".to_string());
-      format!("https://abs.twimg.com/responsive-web/client-web/ondemand.s.{}a.js", on_demand)
-    },
-    _ => format!("https://grok.com/_next/{}", args.xsid_script_id),
-  };
-
-  let xsid_script_body = get_index_page_script_with_client(GetIndexPageScriptArgs {
-    client: &args.client,
-    cookie: args.cookie,
-    script_url: &script_link,
-  }).await?;
-
-  Ok(parse_xsid_script_numbers(&xsid_script_body))
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs};
-  use crate::requests::index_page::get_numbers::{get_numbers, GetNumbersArgs};
-  use crate::requests::index_page::parsers::index::parse_index_verification_token::parse_index_verification_token;
-  use crate::requests::index_page::parsers::script::parse_script_actions_and_xsid_script_path::parse_script_actions_and_xsid_script_path;
-  use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::convert_verification_token_to_loading_anim;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-
-    let page_and_scripts = get_index_page_and_scripts(GetIndexPageAndScriptsArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let verification_token = parse_index_verification_token(&page_and_scripts.index_body_html)
-        .expect("expected verification token");
-
-    println!("Verification Token: {:?}", verification_token);
-
-    let loading_anim = convert_verification_token_to_loading_anim(&verification_token)?;
-
-    println!("Loading Animation: {:?}", loading_anim);
-
-    let actions_and_xsid_script = parse_script_actions_and_xsid_script_path(&page_and_scripts.scripts)?;
-
-    println!("Actions and XSID: {:?}", actions_and_xsid_script);
-
-    let result = get_numbers(GetNumbersArgs {
-      client: &page_and_scripts.client,
-      html: &page_and_scripts.index_body_html,
-      cookie: &cookie,
-      loading_anim: &loading_anim,
-      xsid_script_id: &actions_and_xsid_script.xsid_script_path,
-    }).await?;
-
-    println!("Final Numbers: {:?}", result);
-
-    println!("\n\nBody:\n\n");
-    println!("{}", page_and_scripts.index_body_html);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/get_xsid_script.rs

@@ -1,92 +0,0 @@
-use crate::error::grok_error::GrokError;
-use crate::requests::index_page::parsers::index::parse_index_on_demand_script::parse_index_on_demand_script;
-use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs};
-use wreq::Client;
-
-// self.svg_data, self.numbers = Parser.parse_values(c_request.text, self.anim, self.xsid_script)
-
-pub struct GetXsidScriptArgs<'a> {
-  pub client: &'a Client,
-  pub html: &'a str,
-  pub cookie: &'a str,
-  pub xsid_script_id: &'a str,
-}
-
-/// We pull numbers out of the xsid script
-#[derive(Clone, Debug)]
-pub struct XsidScript {
-  pub xsid_script_body: String,
-}
-
-pub async fn get_xsid_script(args: GetXsidScriptArgs<'_>) -> Result<XsidScript, GrokError> {
-
-  let script_link = match args.xsid_script_id {
-    "ondemand.s" => {
-      // TODO: Not sure this is relevant anymore. Not a full or correct implementation of this branch.
-      let on_demand = parse_index_on_demand_script(&args.html);
-      let on_demand = on_demand.unwrap_or_else(|| "".to_string());
-      format!("https://abs.twimg.com/responsive-web/client-web/ondemand.s.{}a.js", on_demand)
-    },
-    _ => format!("https://grok.com/_next/{}", args.xsid_script_id),
-  };
-
-  let xsid_script_body = get_index_page_script_with_client(GetIndexPageScriptArgs {
-    client: &args.client,
-    cookie: args.cookie,
-    script_url: &script_link,
-  }).await?;
-
-  Ok(XsidScript {
-    xsid_script_body,
-  })
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs};
-  use crate::requests::index_page::get_numbers::{get_numbers, GetNumbersArgs};
-  use crate::requests::index_page::parsers::index::parse_index_verification_token::parse_index_verification_token;
-  use crate::requests::index_page::parsers::script::parse_script_actions_and_xsid_script_path::parse_script_actions_and_xsid_script_path;
-  use crate::requests::index_page::utils::convert_verification_token_to_loading_anim::convert_verification_token_to_loading_anim;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-
-    let page_and_scripts = get_index_page_and_scripts(GetIndexPageAndScriptsArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let verification_token = parse_index_verification_token(&page_and_scripts.index_body_html)
-        .expect("expected verification token");
-
-    println!("Verification Token: {:?}", verification_token);
-
-    let loading_anim = convert_verification_token_to_loading_anim(&verification_token)?;
-
-    println!("Loading Animation: {:?}", loading_anim);
-
-    let actions_and_xsid_script = parse_script_actions_and_xsid_script_path(&page_and_scripts.scripts)?;
-
-    println!("Actions and XSID: {:?}", actions_and_xsid_script);
-
-    let result = get_numbers(GetNumbersArgs {
-      client: &page_and_scripts.client,
-      html: &page_and_scripts.index_body_html,
-      cookie: &cookie,
-      loading_anim: &loading_anim,
-      xsid_script_id: &actions_and_xsid_script.xsid_script_path,
-    }).await?;
-
-    println!("Final Numbers: {:?}", result);
-
-    println!("\n\nBody:\n\n");
-    println!("{}", page_and_scripts.index_body_html);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/mod.rs

@@ -1,8 +0,0 @@
-pub (super) mod requests;
-pub mod get_index_page;
-pub mod get_index_page_and_scripts;
-pub mod get_numbers;
-pub mod get_xsid_script;
-pub mod parsers;
-pub mod signature;
-pub mod utils;

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/mod.rs

@@ -1,8 +0,0 @@
-pub mod parse_index_baggage;
-pub mod parse_index_on_demand_script;
-pub mod parse_index_script_list;
-pub mod parse_index_sentry_trace;
-pub mod parse_index_svg_paths;
-pub mod parse_index_user_email;
-pub mod parse_index_user_id;
-pub mod parse_index_verification_token;

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_baggage.rs

@@ -1,44 +0,0 @@
-use once_cell::sync::Lazy;
-use scraper::{Html, Selector};
-use crate::datatypes::api::baggage::Baggage;
-// <meta name="baggage" content="sentry-environment=production,sentry-public_key=b311e0f2690c81f25e2c4cf6d4f7ce1c,sentry-trace_id=fb5c42c8cff6fd39161dd245154ca599,sentry-org_id=4508179396558848,sentry-sampled=false,sentry-sample_rand=0.7208394686924251,sentry-sample_rate=0"/>
-
-static BAGGAGE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
-  Selector::parse("meta[name=baggage]")
-      .expect("HTML selector should parse")
-});
-
-
-pub fn parse_index_baggage(html: &str) -> Option<Baggage> {
-  let document = Html::parse_document(html);
-  let selected = document.select(&BAGGAGE_SELECTOR);
-  let mut values = selected
-      .filter_map(|node| node.attr("content"))
-      .map(|s| s.to_string())
-      .collect::<Vec<_>>();
-  values.pop().map(|b| Baggage(b))
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_baggage::parse_index_baggage;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let baggage = parse_index_baggage(&index.body);
-
-    println!("Baggage: {:?}", baggage);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_on_demand_script.rs

@@ -1,47 +0,0 @@
-use once_cell::sync::Lazy;
-use regex::Regex;
-
-static ON_DEMAND_REGEX : Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#""ondemand.s":"([^"]*)""#)
-      .expect("Regex should parse")
-});
-
-pub fn parse_index_on_demand_script(html: &str) -> Option<String> {
-  let on_demand = scrape_on_demand_via_regex(html);
-
-  if let Some(on_demand) = on_demand {
-    return Some(on_demand);
-  }
-
-  None
-}
-
-fn scrape_on_demand_via_regex(html: &str) -> Option<String> {
-  let captures = ON_DEMAND_REGEX.captures(html)?;
-  let capture = captures.get(1)?;
-  Some(capture.as_str().to_string())
-}
-
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_on_demand_script::parse_index_on_demand_script;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let result = parse_index_on_demand_script(&index.body);
-    println!("{:?}", result);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_script_list.rs

@@ -1,48 +0,0 @@
-use once_cell::sync::Lazy;
-use scraper::{Html, Selector};
-
-// scripts: list = [s['src'] for s in BeautifulSoup(load_site.text, 'html.parser').find_all('script', src=True) if s['src'].startswith('/_next/static/chunks/')]
-
-static SCRIPT_SELECTOR: Lazy<Selector> = Lazy::new(|| {
-  Selector::parse("script[src]")
-      .expect("HTML selector should parse")
-});
-
-
-pub fn parse_index_svg_paths(html: &str) -> Vec<String> {
-  let document = Html::parse_document(html);
-  let selected = document.select(&SCRIPT_SELECTOR);
-  selected
-      .filter_map(|node| node.attr("src"))
-      .filter(|src| src.starts_with("/_next/static/chunks/"))
-      .map(|s| s.to_string())
-      .collect::<Vec<_>>()
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_script_list::parse_index_svg_paths;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let scripts = parse_index_svg_paths(&index.body);
-
-    println!("Script count: {:?}", scripts.len());
-
-    for script in scripts {
-      println!("{}", script);
-    }
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_sentry_trace.rs

@@ -1,85 +0,0 @@
-use crate::datatypes::api::sentry_trace::SentryTrace;
-use once_cell::sync::Lazy;
-use scraper::{Html, Selector};
-
-/// eg. <meta name="baggage" content="sentry-environment=production,sentry-public_key=b311e0f2690c81f25e2c4cf6d4f7ce1c,sentry-trace_id=fb5c42c8cff6fd39161dd245154ca599,sentry-org_id=4508179396558848,sentry-sampled=false,sentry-sample_rand=0.7208394686924251,sentry-sample_rate=0"/>
-static SENTRY_TRACE_SELECTOR : Lazy<Selector> = Lazy::new(|| {
-  Selector::parse("meta[name=sentry-trace]")
-      .expect("HTML selector should parse")
-});
-
-
-/// Extract the sentry trace, then process it.
-/// The Python library discards the latter half of the trace.
-pub fn parse_index_sentry_trace(html: &str) -> Option<SentryTrace> {
-  get_meta_value(html)
-      .map(|trace| split_sentry_trace(&trace))
-      .map(|trace| SentryTrace(trace))
-}
-
-fn get_meta_value(html: &str) -> Option<String> {
-  let document = Html::parse_document(html);
-  let selected = document.select(&SENTRY_TRACE_SELECTOR);
-  let mut values = selected
-      .filter_map(|node| node.attr("content"))
-      .map(|s| s.to_string())
-      .collect::<Vec<_>>();
-  values.pop()
-}
-
-fn split_sentry_trace(trace: &str) -> String {
-  trace.split("-")
-      .next()
-      .map(|v| v.to_string())
-      .unwrap_or_else(|| trace.to_string())
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_sentry_trace::parse_index_sentry_trace;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  mod test_request {
-    use super::*;
-
-    #[tokio::test]
-    #[ignore] // Manual test invocation
-    async fn test() -> AnyhowResult<()> {
-      let cookie = get_test_cookies()?;
-      let index = get_index(GetIndexPageArgs {
-        cookie: &cookie,
-      }).await?;
-
-      let sentry_trace = parse_index_sentry_trace(&index.body);
-
-      println!("Sentry trace: {:?}", sentry_trace);
-
-      assert_eq!(1, 2);
-      Ok(())
-    }
-  }
-
-  mod test_sentry_split {
-    use crate::requests::index_page::parsers::index::parse_index_sentry_trace::split_sentry_trace;
-
-    #[test]
-    fn test_production_case() {
-      let result = split_sentry_trace("d6d7c55e4a489c0dabaed06e5c7b257b-d2391b601d54494f-0");
-      assert_eq!(&result, "d6d7c55e4a489c0dabaed06e5c7b257b");
-    }
-
-    #[test]
-    fn fallback_case_1() {
-      let result = split_sentry_trace("asdf");
-      assert_eq!(&result, "asdf");
-    }
-
-    #[test]
-    fn fallback_case_2() {
-      let result = split_sentry_trace("");
-      assert_eq!(&result, "");
-    }
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_svg_paths.rs

@@ -1,52 +0,0 @@
-use crate::datatypes::api::svg_path_data::SvgPathData;
-use once_cell::sync::Lazy;
-use scraper::{Html, Selector};
-
-static SVG_PATH_SELECTOR: Lazy<Selector> = Lazy::new(|| {
-  Selector::parse("path[d]")
-      .expect("HTML selector should parse")
-});
-
-
-pub fn parse_svg_paths_from_index_html(html: &str) -> Vec<SvgPathData> {
-  // From https://github.com/realasfngl/Grok-Api :
-  //   all_d_values = findall(r'"d":"(M[^"]{200,})"', html)
-  //   svg_data = all_d_values[int(loading.split("loading-x-anim-")[1])]
-
-  let document = Html::parse_document(html);
-  let selected = document.select(&SVG_PATH_SELECTOR);
-  selected
-      .filter_map(|node| node.attr("d"))
-      .filter(|path_data| path_data.len() >= 200) // NB: Not sure why these SVG strokes are more important.
-      .map(|s| SvgPathData(s.to_string()))
-      .collect::<Vec<_>>()
-}
-
-// TODO: Get from JSON instead ? - That's how the python source does it.
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_svg_paths::parse_svg_paths_from_index_html;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let paths = parse_svg_paths_from_index_html(&index.body);
-    println!("Paths count: {:?}", paths.len());
-
-    for path in paths {
-      println!("{}", path.0);
-    }
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_user_email.rs

@@ -1,50 +0,0 @@
-use crate::datatypes::api::user_email::UserEmail;
-use once_cell::sync::Lazy;
-use regex::Regex;
-
-/// Find the user email in the index.html payload
-/// eg.
-static JSON_REGEX: Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#"\\?"email\\?":\s*\\?"([^"\\]+)\\?",?"#)
-      .expect("Regex should parse")
-});
-
-/// Parse the user email from the index.html
-pub fn parse_index_user_email(html: &str) -> Option<UserEmail> {
-  let maybe_meta = scrape_user_email_via_regex(html);
-
-  if let Some(meta) = maybe_meta {
-    return Some(UserEmail(meta))
-  }
-
-  None
-}
-
-fn scrape_user_email_via_regex(html: &str) -> Option<String> {
-  let captures = JSON_REGEX.captures(html)?;
-  let capture = captures.get(1)?;
-  Some(capture.as_str().to_string())
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_user_email::parse_index_user_email;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let user_email = parse_index_user_email(&index.body);
-    println!("User Email : {:?}", user_email);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_user_id.rs

@@ -1,50 +0,0 @@
-use crate::datatypes::api::user_id::UserId;
-use once_cell::sync::Lazy;
-use regex::Regex;
-
-/// Find the user id in the index.html payload
-/// eg. \"userId\":\"85980643-ffab-4984-a3de-59a608c47d7f\",
-static JSON_REGEX: Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#"\\?"userI[dD]\\?":\s*\\?"([a-zA-Z0-9-]{25,})\\?",?"#)
-      .expect("Regex should parse")
-});
-
-/// Parse the user id from the index.html
-pub fn parse_index_user_id(html: &str) -> Option<UserId> {
-  let maybe_meta = scrape_user_id_via_regex(html);
-
-  if let Some(meta) = maybe_meta {
-    return Some(UserId(meta))
-  }
-
-  None
-}
-
-fn scrape_user_id_via_regex(html: &str) -> Option<String> {
-  let captures = JSON_REGEX.captures(html)?;
-  let capture = captures.get(1)?;
-  Some(capture.as_str().to_string())
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_user_id::parse_index_user_id;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let user_id = parse_index_user_id(&index.body);
-    println!("User ID : {:?}", user_id);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_verification_token.rs

@@ -1,80 +0,0 @@
-use crate::datatypes::api::verification_token::VerificationToken;
-use once_cell::sync::Lazy;
-use regex::Regex;
-use scraper::{Html, Selector};
-
-/// Find the verification token in the meta tags using HTML parsing.
-static META_SELECTOR : Lazy<Selector> = Lazy::new(|| {
-  Selector::parse("meta[name=grok-site-verification]")
-      .expect("HTML selector should parse")
-});
-
-/// Find the verification token in the meta tags using regex
-// eg. <meta name="grok-site-verification" content="iFlTOEJNfQZP1B6YMRf/zuj3eYFrKWi6eNUg5XvvpllTOw5TS82coZkeUBdiHxYr"/>
-static META_TAG_REGEX : Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#""grok-site-verification"\s*content="([A-Za-z0-9/=+-]{5,})""#)
-      .expect("Regex should parse")
-});
-
-/// Find the verification token in the script tags using regex
-/// eg. {\"name\":\"grok-site-verification\",\"content\":\"iFlTOEJNfQZP1B6YMRf/zuj3eYFrKWi6eNUg5XvvpllTOw5TS82coZkeUBdiHxYr\"}]
-static JSON_REGEX: Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#""grok-site-verification\\?",\s*\\?"content\\?":\\?"([A-Za-z0-9/=+-]{5,})\\?""#)
-      .expect("Regex should parse")
-});
-
-/// Parse the verification token from the root HTML
-pub fn parse_index_verification_token(html: &str) -> Option<VerificationToken> {
-  let maybe_meta = scrape_meta_tag_via_parsing(html);
-
-  if let Some(meta) = maybe_meta {
-    return Some(VerificationToken(meta))
-  }
-
-  let maybe_script = scrape_script_via_regex(html);
-
-  if let Some(script) = maybe_script {
-    return Some(VerificationToken(script))
-  }
-
-  None
-}
-
-fn scrape_script_via_regex(html: &str) -> Option<String> {
-  let captures = META_TAG_REGEX.captures(html)?;
-  let capture = captures.get(1)?;
-  Some(capture.as_str().to_string())
-}
-
-fn scrape_meta_tag_via_parsing(html: &str) -> Option<String> {
-  let document = Html::parse_document(html);
-  let selected = document.select(&META_SELECTOR);
-  let mut values = selected
-      .filter_map(|node| node.attr("content"))
-      .map(|s| s.to_string())
-      .collect::<Vec<_>>();
-  values.pop()
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};
-  use crate::requests::index_page::parsers::index::parse_index_verification_token::parse_index_verification_token;
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // Manual test invocation
-  async fn test() -> AnyhowResult<()> {
-    let cookie = get_test_cookies()?;
-    let index = get_index(GetIndexPageArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let result = parse_index_verification_token(&index.body);
-    println!("{:?}", result);
-    
-    assert_eq!(1, 2);
-    Ok(())
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/mod.rs

@@ -1,2 +0,0 @@
-pub mod index;
-pub mod script;

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/script/mod.rs

@@ -1,2 +0,0 @@
-pub mod parse_xsid_script_numbers;
-pub mod parse_script_actions_and_xsid_script_path;

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/script/parse_script_actions_and_xsid_script_path.rs

@@ -1,124 +0,0 @@
-use crate::error::grok_client_error::GrokClientError;
-use once_cell::sync::Lazy;
-use regex::Regex;
-use std::collections::HashMap;
-use log::info;
-
-static ACTIONS_REGEX : Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#"createServerReference\)\("([a-f0-9]+)""#)
-      .expect("Regex should parse")
-});
-
-/// 2025-10-01: "(static/chunks/[^"]+\.js)"[^}]*?a\(880932\)"
-/// 2025-12-01: "(static/chunks/[^"]+\.js)"[^}]*?\(880932\)"
-static XSID_SCRIPT_REGEX : Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#""(static/chunks/[^"]+\.js)"[^}]*?\(880932\)"#)
-      .expect("Regex should parse")
-});
-
-#[derive(Debug, Clone)]
-pub struct ActionsAndXsid {
-  /// A list of opaque IDs for "actions"
-  pub actions: Vec<String>,
-
-  /// The URL path of the xsid script, with no leading slash.
-  pub xsid_script_path: String,
-}
-
-pub fn parse_script_actions_and_xsid_script_path(scripts: &HashMap<String, String>) -> Result<ActionsAndXsid, GrokClientError> {
-  let mut action_script_path = None;
-  let mut script_content_1 = None;
-  let mut script_content_2 = None;
-
-  for (script_path, script_body) in scripts.iter() {
-    if script_body.contains("anonPrivateKey") {
-      action_script_path = Some(script_path.to_string());
-      script_content_1 = Some(script_body.to_string());
-    } else if script_body.contains("880932") {
-      script_content_2 = Some(script_body.to_string());
-    }
-  }
-
-  // NB: Commented out because we don't use it.
-  // let action_script_path = match action_script_path {
-  //   Some(action_script_path) => action_script_path,
-  //   None => return Err(GrokClientError::ScriptLogicOutOfDate),
-  // };
-
-  let script_content_1 = match script_content_1 {
-    Some(script_content_1) => script_content_1,
-    None => return Err(GrokClientError::Script1LogicOutOfDate),
-  };
-
-  let script_content_2 = match script_content_2 {
-    Some(script_content_2) => script_content_2,
-    None => return Err(GrokClientError::Script2LogicOutOfDate),
-  };
-
-  //println!("Script content 1:");
-  //println!("{}", script_content_1);
-  //println!("Script content 2:");
-  //println!("{}", script_content_2);
-
-  let actions = ACTIONS_REGEX.captures_iter(&script_content_1)
-      .flat_map(|captures| captures.get(1).map(|m| m.as_str().to_string()))
-      .collect::<Vec<_>>();
-
-  let xsid_script_path = parse_xsid_script_path_from_turbopack_script(&script_content_2)?;
-
-  Ok(ActionsAndXsid {
-    actions,
-    xsid_script_path,
-  })
-}
-
-fn parse_xsid_script_path_from_turbopack_script(script_source: &str) -> Result<String, GrokClientError> {
-  let xsid_script_path = XSID_SCRIPT_REGEX.captures(&script_source)
-      .map(|captures| captures.get(1)
-          .map(|m| m.as_str()
-              .to_string()))
-      .flatten();
-
-  match xsid_script_path {
-    Some(xsid_script) => Ok(xsid_script),
-    None => Err(GrokClientError::ScriptXsidLogicOutOfDate),
-  }
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::get_index_page_and_scripts::{get_index_page_and_scripts, GetIndexPageAndScriptsArgs};
-  use crate::requests::index_page::parsers::script::parse_script_actions_and_xsid_script_path::{parse_script_actions_and_xsid_script_path, parse_xsid_script_path_from_turbopack_script};
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn test() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-    let cookie = get_test_cookies()?;
-    let page_and_scripts = get_index_page_and_scripts(GetIndexPageAndScriptsArgs {
-      cookie: &cookie,
-    }).await?;
-
-    let result = parse_script_actions_and_xsid_script_path(&page_and_scripts.scripts)?;
-
-    println!("{:?}", result);
-
-    assert_eq!(1, 2);
-    Ok(())
-  }
-
-  #[test]
-  fn parse_turbopack_script_2_for_xsid() {
-    // NB: The XSID parsing script broke on December 1st.
-    // The valid script to parse out is "static/chunks/6450f465e55b5c24.js" (as it contains the magic numbers). This comes before the string (880932)
-    // They probably just changed the TurboPack configs.
-    let example_script_2 = r#"
-      (globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,483347,a=>{a.v(s=>Promise.all(["static/chunks/6450f465e55b5c24.js"].map(s=>a.l(s))).then(()=>s(880932)))},330040,a=>{a.v(s=>Promise.all(["static/chunks/5cc0fc78839e38d4.js"].map(s=>a.l(s))).then(()=>s(921398)))},127660,a=>{a.v(s=>Promise.all(["static/chunks/fc24fa4ac57f0fa9.js"].map(s=>a.l(s))).then(()=>s(969999)))},307492,a=>{a.v(s=>Promise.all(["static/chunks/eb46b9f393cc69d9.js"].map(s=>a.l(s))).then(()=>s(529107)))},460973,a=>{a.v(s=>Promise.all(["static/chunks/7ff9ad06da182f94.js"].map(s=>a.l(s))).then(()=>s(296273)))},900864,
-    "#;
-
-    let parsed = parse_xsid_script_path_from_turbopack_script(example_script_2).unwrap();
-    assert_eq!(parsed, "static/chunks/6450f465e55b5c24.js".to_string());
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/script/parse_xsid_script_numbers.rs

@@ -1,27 +0,0 @@
-use crate::datatypes::api::xsid_numbers::XsidNumbers;
-use log::error;
-use once_cell::sync::Lazy;
-use regex::Regex;
-use std::str::FromStr;
-
-static NUMBERS_REGEX : Lazy<Regex> = Lazy::new(|| {
-  Regex::new(r#"x\[(\d+)\]\s*,\s*16"#)
-      .expect("Regex should parse")
-});
-
-pub fn parse_xsid_script_numbers(script_body: &str) -> XsidNumbers {
-  let numbers = NUMBERS_REGEX.captures_iter(&script_body)
-      .flat_map(|captures| captures.get(1).map(|m| m.as_str().to_string()))
-      .flat_map(|num| match u32::from_str(&num) {
-        Ok(num) => Some(num),
-        Err(err) => {
-          error!("Error parsing: {:?}", err);
-          None
-        }
-      })
-      .collect::<_>();
-  
-  XsidNumbers {
-    numbers,
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/requests/get_index_page_script_with_client.rs

@@ -1,63 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use log::warn;
-use wreq::Client;
-
-const INDEX_URL: &str = "https://grok.com";
-
-pub struct GetIndexPageScriptArgs<'a> {
-  pub client: &'a Client,
-  pub cookie: &'a str,
-  pub script_url: &'a str,
-}
-
-
-/// Get javascript that we'll need for client crypto purposes.
-pub async fn get_index_page_script_with_client(args: GetIndexPageScriptArgs<'_>) -> Result<String, GrokError> {
-
-  let script_url;
-
-  if args.script_url.starts_with("https://") {
-    script_url = args.script_url.to_string();
-  } else {
-    script_url = get_script_url(args.script_url);
-  }
-
-  let builder = args.client.get(script_url)
-      .header("User-Agent", FIREFOX_143_MAC_USER_AGENT)
-      .header("Accept", "*/*")
-      .header("Accept-Language", "en-US,en;q=0.5")
-      .header("Accept-Encoding", "gzip, deflate, br, zstd")
-      .header("Referer", "https://grok.com")
-      .header("Connection", "keep-alive")
-      .header("Sec-Fetch-Dest", "script")
-      .header("Sec-Fetch-Mode", "no-cors")
-      .header("Sec-Fetch-Site", "same-origin")
-      .header("TE", "trailers");
-
-  let response = builder.send()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-  
-  let status = response.status();
-
-  // TODO: Cloudflare handling.
-  let body = response.text()
-      .await
-      .map_err(|err| GrokClientError::WreqClientError(err))?;
-
-  if !status.is_success() {
-    warn!("could not get index page script: {}, body: {}", status, body);
-  }
-
-  Ok(body)
-}
-
-fn get_script_url(script_path: &str) -> String {
-  if script_path.starts_with("/") {
-    format!("{}{}", INDEX_URL, script_path)
-  } else {
-    format!("{}/{}", INDEX_URL, script_path)
-  }
-}

crates/api_clients/grok_consumer_client/src/requests/index_page/requests/get_index_page_scripts_with_client.rs

@@ -1,80 +0,0 @@
-use crate::client::browser_user_agents::FIREFOX_143_MAC_USER_AGENT;
-use crate::error::grok_client_error::GrokClientError;
-use crate::error::grok_error::GrokError;
-use crate::requests::index_page::requests::get_index_page_script_with_client::{get_index_page_script_with_client, GetIndexPageScriptArgs};
-use log::error;
-use std::collections::HashMap;
-use wreq::Client;
-
-pub struct GetIndexPageScriptsArgs<'a> {
-  pub client: &'a Client,
-  pub cookie: &'a str,
-  pub scripts: &'a Vec<String>,
-}
-
-/// Get javascript that we'll need for client crypto purposes.
-pub async fn get_index_page_scripts_with_client(args: GetIndexPageScriptsArgs<'_>) -> Result<HashMap<String, String>, GrokError> {
-  let mut map = HashMap::with_capacity(args.scripts.len());
-
-  for script_url in args.scripts {
-    let result = get_index_page_script_with_client(GetIndexPageScriptArgs {
-      client: args.client,
-      cookie: args.cookie,
-      script_url,
-    }).await;
-
-    let script = match result {
-      Ok(script) => script,
-      Err(err) => {
-        error!("Error fetching script: {}: {:?}", script_url, err);
-        return Err(err);
-      }
-    };
-
-    map.insert(script_url.to_string(), script);
-  }
-
-  Ok(map)
-}
-
-#[cfg(test)]
-mod tests {
-  use crate::requests::index_page::parsers::index::parse_index_script_list::parse_index_svg_paths;
-  use crate::requests::index_page::requests::get_index_page_scripts_with_client::{get_index_page_scripts_with_client, GetIndexPageScriptsArgs};
-  use crate::requests::index_page::requests::get_index_page_with_client::{get_index_page_with_client, GetIndexPageWithClientArgs};
-  use crate::test_utils::get_test_cookies::get_test_cookies;
-  use crate::utils::create_firefox_client::create_firefox_client;
-  use errors::AnyhowResult;
-
-  #[tokio::test]
-  #[ignore] // manually test
-  async fn test() -> AnyhowResult<()> {
-    //setup_test_logging(LevelFilter::Trace);
-
-    let client = create_firefox_client()?;
-    let cookie = get_test_cookies()?;
-
-    let index = get_index_page_with_client(GetIndexPageWithClientArgs {
-      client: &client,
-      cookie: &cookie,
-    }).await?;
-
-    let scripts = parse_index_svg_paths(&index.body);
-
-    let scripts = get_index_page_scripts_with_client(GetIndexPageScriptsArgs {
-      client: &client,
-      cookie: &cookie,
-      scripts: &scripts,
-    }).await?;
-
-    for (k, v) in scripts.iter() {
-      let mut body = v.to_string();
-      body.truncate(100);
-      println!("{}: {}\n\n", k, body);
-    }
-
-    assert_eq!(1, 2);
-
-    Ok(())
-  }
-}