diff --git a/.github/workflows/tauri-publish-2d.yml b/.github/workflows/tauri-publish-2d.yml new file mode 100644 index 0000000000..2d36265575 --- /dev/null +++ b/.github/workflows/tauri-publish-2d.yml @@ -0,0 +1,63 @@ +name: 'publish' + +on: + push: + branches: + - tauri-prod-release + +# This workflow will trigger on each push to the `release` branch to create or update a GitHub release, build your app, and upload the artifacts to the release. + +jobs: + publish-tauri: + permissions: + contents: write + strategy: + fail-fast: false + matrix: + include: + - platform: 'macos-latest' # for Arm based macs (M1 and above). + args: '--target aarch64-apple-darwin' + # - platform: 'macos-latest' # for Intel based macs. + # args: '--target x86_64-apple-darwin' + # - platform: 'ubuntu-22.04' # for Tauri v1 you could replace this with ubuntu-20.04. + # args: '' + # - platform: 'windows-latest' + # args: '' + + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v4 + + - name: setup node + uses: actions/setup-node@v4 + with: + node-version: lts/* + + - name: install Rust stable + uses: dtolnay/rust-toolchain@stable + with: + # Those targets are only used on macos runners so it's in an `if` to slightly speed up windows and linux builds. + targets: ${{ matrix.platform == 'macos-latest' && 'aarch64-apple-darwin,x86_64-apple-darwin' || '' }} + + - name: install dependencies (ubuntu only) + if: matrix.platform == 'ubuntu-22.04' # This must match the platform value defined above. + run: | + sudo apt-get update + sudo apt-get install -y libwebkit2gtk-4.0-dev libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf + # webkitgtk 4.0 is for Tauri v1 - webkitgtk 4.1 is for Tauri v2. + # You can remove the one that doesn't apply to your app to speed up the workflow a bit. + + - name: install frontend dependencies + run: cd ./crates/desktop/tauri-artcraft && yarn install # change this to npm, pnpm or bun depending on which one you use. + + - uses: tauri-apps/tauri-action@v0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + projectPath: ./crates/desktop/tauri-artcraft + tagName: app-v__VERSION__ # the action automatically replaces \_\_VERSION\_\_ with the app version. + releaseName: 'App v__VERSION__' + releaseBody: 'See the assets to download this version and install.' + releaseDraft: true + prerelease: false + args: ${{ matrix.args }} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 47e7ca4634..5219edda8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5653,8 +5653,10 @@ dependencies = [ name = "images" version = "0.0.1" dependencies = [ + "anyhow", "errors", "image", + "mimetypes", "testing", "workspace-build-acceleration", ] @@ -6868,7 +6870,9 @@ dependencies = [ name = "mimetypes" version = "0.0.1" dependencies = [ - "infer 0.16.0", + "anyhow", + "infer 0.19.0", + "testing", "workspace-build-acceleration", ] @@ -7844,12 +7848,14 @@ dependencies = [ "anyhow", "base64 0.22.1", "chrono", + "cookie 0.18.1", "crockford", "errors", "filesys", "hex-literal", "idempotency", "log", + "once_cell", "rand 0.9.0", "reqwest 0.12.15", "serde", @@ -10633,6 +10639,7 @@ dependencies = [ "log", "openai_sora_client", "r2d2_redis", + "reqwest 0.12.15", "workspace-build-acceleration", ] @@ -11188,6 +11195,7 @@ dependencies = [ "hostname 0.4.0", "http_server_common", "idempotency", + "images", "infer 0.16.0", "language-tags", "lexical-sort", diff --git a/Cargo.toml b/Cargo.toml index 0a700a74f0..170792f44f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,6 +155,7 @@ workspace-build-acceleration = { path = "crates/workspace-build-acceleration" } anyhow = "1.0.97" base64 = "0.22.1" chrono = { version = "0.4.40", features = ["serde"] } +cookie = "0.18.1" hex-literal = "1.0.0" log = "0.4.27" once_cell = "1.21.3" @@ -176,7 +177,9 @@ enums = { path = "crates/_types/enums" } enums_public = { path = "crates/_types/enums_public" } errors = { path = "crates/lib/errors" } filesys = { path = "crates/lib/files/filesys" } +images = { path = "crates/lib/files/images" } memory_caching = { path = "crates/lib/caching/memory_caching" } +mimetypes = { path = "crates/lib/files/mimetypes" } openai_sora_client = { path = "crates/lib/api_clients/openai_sora_client" } primitives = { path = "crates/lib/data/primitives" } shared_service_components = { path = "crates/lib/shared_service_components" } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs index 66b24a7c05..f46dc5f811 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs @@ -28,11 +28,12 @@ pub enum MediaFileOriginCategory { DeviceApi, /// From Storyteller Studio Engine + #[deprecated(note="This db field should only denote file provenance, not the product.")] #[serde(rename = "studio")] StorytellerStudio, /// From Storyteller Studio Engine - #[deprecated(note = "Use StorytellerStudio instead")] + #[deprecated(note = "DO NOT USE. Originally deprecated in favor of `StorytellerStudio`.")] StoryEngine, } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 7813f50ba7..8ffc554344 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -39,6 +39,10 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "vst")] VideoStyleTransfer, + /// Image Studio + #[serde(rename = "image_studio")] + ImageStudio, + /// Storyteller Studio #[serde(rename = "studio")] StorytellerStudio, @@ -87,6 +91,7 @@ impl MediaFileOriginProductCategory { Self::FaceFusion => "face_fusion", Self::FaceMirror => "face_mirror", Self::VideoStyleTransfer => "vst", + Self::ImageStudio => "image_studio", Self::StorytellerStudio => "studio", Self::TextToSpeech => "tts", Self::VoiceConversion => "voice_conversion", @@ -105,6 +110,7 @@ impl MediaFileOriginProductCategory { "face_fusion" => Ok(Self::FaceFusion), "face_mirror" => Ok(Self::FaceMirror), "vst" => Ok(Self::VideoStyleTransfer), + "image_studio" => Ok(Self::ImageStudio), "studio" => Ok(Self::StorytellerStudio), "tts" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), @@ -126,6 +132,7 @@ impl MediaFileOriginProductCategory { Self::FaceFusion, Self::FaceMirror, Self::VideoStyleTransfer, + Self::ImageStudio, Self::StorytellerStudio, Self::TextToSpeech, Self::VoiceConversion, @@ -153,6 +160,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::FaceFusion, "face_fusion"); assert_serialization(MediaFileOriginProductCategory::FaceMirror, "face_mirror"); assert_serialization(MediaFileOriginProductCategory::VideoStyleTransfer, "vst"); + assert_serialization(MediaFileOriginProductCategory::ImageStudio, "image_studio"); assert_serialization(MediaFileOriginProductCategory::StorytellerStudio, "studio"); assert_serialization(MediaFileOriginProductCategory::TextToSpeech, "tts"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); @@ -170,6 +178,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::FaceFusion.to_str(), "face_fusion"); assert_eq!(MediaFileOriginProductCategory::FaceMirror.to_str(), "face_mirror"); assert_eq!(MediaFileOriginProductCategory::VideoStyleTransfer.to_str(), "vst"); + assert_eq!(MediaFileOriginProductCategory::ImageStudio.to_str(), "image_studio"); assert_eq!(MediaFileOriginProductCategory::StorytellerStudio.to_str(), "studio"); assert_eq!(MediaFileOriginProductCategory::TextToSpeech.to_str(), "tts"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); @@ -187,6 +196,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("face_fusion").unwrap(), MediaFileOriginProductCategory::FaceFusion); assert_eq!(MediaFileOriginProductCategory::from_str("face_mirror").unwrap(), MediaFileOriginProductCategory::FaceMirror); assert_eq!(MediaFileOriginProductCategory::from_str("vst").unwrap(), MediaFileOriginProductCategory::VideoStyleTransfer); + assert_eq!(MediaFileOriginProductCategory::from_str("image_studio").unwrap(), MediaFileOriginProductCategory::ImageStudio); assert_eq!(MediaFileOriginProductCategory::from_str("studio").unwrap(), MediaFileOriginProductCategory::StorytellerStudio); assert_eq!(MediaFileOriginProductCategory::from_str("tts").unwrap(), MediaFileOriginProductCategory::TextToSpeech); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); @@ -200,12 +210,13 @@ mod tests { #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 13); + assert_eq!(variants.len(), 14); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceFusion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceMirror)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoStyleTransfer)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ImageStudio)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::StorytellerStudio)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); diff --git a/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_generation_command.rs b/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_generation_command.rs index 516dcb6a05..a67bfaed0e 100644 --- a/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_generation_command.rs +++ b/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_generation_command.rs @@ -8,10 +8,10 @@ use image::codecs::png::{CompressionType, FilterType, PngEncoder}; use image::{DynamicImage, ImageReader}; use log::{error, info}; use openai_sora_client::credentials::SoraCredentials; -use openai_sora_client::image_gen::common::{ImageSize, NumImages}; -use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; -use openai_sora_client::upload::upload_media_from_bytes::sora_media_upload_from_bytes; -use openai_sora_client::upload::upload_media_from_file::SoraMediaUploadRequest; +use openai_sora_client::creds::credential_migration::CredentialMigrationRef; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; use std::fs::read_to_string; use std::io::Cursor; use tauri::{AppHandle, Manager, State}; @@ -43,14 +43,12 @@ pub async fn generate_image( sora_creds_holder: &SoraCredentialHolder, ) -> AnyhowResult<()> { - let sora_credentials = read_sora_credentials_from_disk(app_data_root) + let creds = read_sora_credentials_from_disk(app_data_root) .map_err(|err| { error!("Failed to read Sora credentials from disk: {:?}", err); err })?; - //let sora_credentials = sora_creds_holder.get_credentials()?; - let mut sora_media_tokens = vec![]; if let Some(image) = maybe_image { @@ -58,7 +56,7 @@ pub async fn generate_image( let filename = "image.png".to_string(); - let response = sora_media_upload_from_bytes(image_bytes, filename, &sora_credentials) + let response = sora_media_upload_from_bytes(image_bytes, filename, CredentialMigrationRef::New(&creds)) .await .map_err(|err| { error!("Failed to upload image to Sora: {:?}", err); @@ -73,7 +71,7 @@ pub async fn generate_image( num_images: NumImages::One, image_size: ImageSize::Square, sora_media_tokens: sora_media_tokens.clone(), - credentials: &sora_credentials, + credentials: CredentialMigrationRef::New(&creds), }).await .map_err(|err| { error!("Failed to call Sora image generation: {:?}", err); diff --git a/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_remix_command.rs b/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_remix_command.rs index 8e3d0d3746..a032a2f872 100644 --- a/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_remix_command.rs +++ b/crates/desktop/tauri-artcraft/src/commands/sora/sora_image_remix_command.rs @@ -11,11 +11,14 @@ use image::codecs::png::{CompressionType, FilterType, PngEncoder}; use image::{DynamicImage, ImageReader}; use log::{debug, error, info}; use openai_sora_client::credentials::SoraCredentials; -use openai_sora_client::image_gen::common::{ImageSize, NumImages}; -use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; -use openai_sora_client::sentinel_refresh::refresh_sentinel; -use openai_sora_client::upload::upload_media_from_bytes::sora_media_upload_from_bytes; -use openai_sora_client::upload::upload_media_from_file::{sora_media_upload_from_file, SoraMediaUploadRequest}; +use openai_sora_client::creds::credential_migration::CredentialMigrationRef; +use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; +use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; +use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file; use serde_derive::Deserialize; use std::fs::read_to_string; use std::io::Cursor; @@ -50,6 +53,8 @@ pub async fn sora_image_remix_command( ) -> Result { info!("image_generation_command called; processing image..."); + // TODO(bt,2025-04-24): Better error messages to caller + generate_image(request, &app_data_root, &sora_creds_manager) .await .map_err(|err| { @@ -80,77 +85,62 @@ pub async fn generate_image( let files_to_upload = vec![filename]; - // TODO(bt,2025-04-21): Read from in-memory cache instead, but allow for desktop replacement. - let mut sora_creds = read_sora_credentials_from_disk(app_data_root) - .map_err(|err| { - error!("Failed to read Sora credentials from disk: {:?}", err); - err - })?; - let no_sentinel = !app_data_root.get_sora_sentinel_file_path().is_file() - || sora_creds.sentinel.is_none(); - if no_sentinel { - info!("Refreshing credentials..."); + let mut creds = sora_creds_manager.get_credentials_required()?; - sora_creds = sora_creds_manager.call_sentinel_refresh() - .await - .map_err(|err| { - error!("Failed to refresh: {:?}", err); - err - })?; + let credential_updated = maybe_upgrade_or_renew_session(&mut creds) + .await + .map_err(|err| { + error!("Failed to upgrade or renew session: {:?}", err); + err + })?; - // TODO: Write to disk. + if credential_updated { + info!("Storing updated credentials"); + sora_creds_manager.set_credentials(&creds)?; } let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len()); - for file_path in files_to_upload { - let sora_upload_response = sora_media_upload_from_file(file_path, &sora_creds) - .await?; + for (i, file_path) in files_to_upload.iter().enumerate() { + info!("Uploading image {} of {}...", (i+1), files_to_upload.len()); - sora_media_tokens.push(sora_upload_response.id); + let (response, maybe_new_credentials) = + image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest { + file_path, + credentials: &creds, + }).await?; + + if let Some(new_creds) = maybe_new_credentials { + info!("Storing updated credentials."); + sora_creds_manager.set_credentials(&new_creds)?; + creds = new_creds; + } + + sora_media_tokens.push(response.id); } + info!("Calling image generation..."); + // TODO(bt,2025-04-21): Download media tokens. // Note: This is incredibly inefficient. We should keep a local cache. // Also, if they've already been uploaded to OpenAI, we shouldn't continue to re-upload. - let mut response = sora_image_gen_remix(SoraImageGenRemixRequest { - prompt: request.prompt.to_string(), - num_images: NumImages::One, - image_size: ImageSize::Square, - sora_media_tokens: sora_media_tokens.clone(), - credentials: &sora_creds, - }).await; + let (response, maybe_new_creds) = + image_remix_with_session_auto_renew(ImageRemixAutoRenewRequest { + prompt: request.prompt.to_string(), + num_images: NumImages::One, + image_size: ImageSize::Square, + sora_media_tokens: sora_media_tokens.clone(), + credentials: &creds, + }).await?; - if let Err(err) = &response { - error!("Error in generating image: {:?}", err); - - sora_creds = sora_creds_manager.call_sentinel_refresh() - .await - .map_err(|err| { - error!("Failed to refresh: {:?}", err); - err - })?; - - info!("Retrying request..."); - - response = sora_image_gen_remix(SoraImageGenRemixRequest { - prompt: request.prompt.to_string(), - num_images: NumImages::One, - image_size: ImageSize::Square, - sora_media_tokens: sora_media_tokens.clone(), - credentials: &sora_creds, - }).await; + if let Some(new_creds) = maybe_new_creds { + info!("Storing updated credentials."); + sora_creds_manager.set_credentials(&new_creds)?; } - let response = response - .map_err(|err| { - error!("Failed to call Sora image generation: {:?}", err); - err - })?; - println!(">> TASK ID: {:?} ", response.task_id); Ok(()) diff --git a/crates/desktop/tauri-artcraft/src/state/sora/mod.rs b/crates/desktop/tauri-artcraft/src/state/sora/mod.rs index 0726ea796b..2038c13e56 100644 --- a/crates/desktop/tauri-artcraft/src/state/sora/mod.rs +++ b/crates/desktop/tauri-artcraft/src/state/sora/mod.rs @@ -1,3 +1,4 @@ pub mod read_sora_credentials_from_disk; +pub mod read_sora_credentials_from_disk_with_upgrade; pub mod sora_credential_holder; pub mod sora_credential_manager; \ No newline at end of file diff --git a/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk.rs b/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk.rs index e356b0ee7c..9d4579249c 100644 --- a/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk.rs +++ b/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk.rs @@ -1,36 +1,48 @@ -use std::fs::read_to_string; -use errors::AnyhowResult; -use openai_sora_client::credentials::SoraCredentials; use crate::state::app_dir::AppDataRoot; +use anyhow::anyhow; +use errors::AnyhowResult; +use openai_sora_client::creds::sora_cookies::SoraCookies; +use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; +use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use openai_sora_client::creds::sora_sentinel::SoraSentinel; +use std::fs::read_to_string; -pub fn read_sora_credentials_from_disk(app_data_root: &AppDataRoot) -> AnyhowResult { +pub fn read_sora_credentials_from_disk(app_data_root: &AppDataRoot) -> AnyhowResult { let cookie_file = app_data_root.get_sora_cookie_file_path(); let bearer_file = app_data_root.get_sora_bearer_token_file_path(); let sentinel_file = app_data_root.get_sora_sentinel_file_path(); - let cookie = read_to_string(cookie_file)? + if !cookie_file.exists() { + return Err(anyhow!("Cookie file does not exist: {:?}", cookie_file)); + } + + let value = read_to_string(cookie_file)? .trim() .to_string(); - let bearer = read_to_string(bearer_file)? - .trim() - .to_string(); + let cookie = SoraCookies::new(value); - if sentinel_file.exists() && sentinel_file.is_file() { - let sentinel = read_to_string(sentinel_file)? + let mut bearer = None; + let mut sentinel = None; + + if bearer_file.exists() { + let value = read_to_string(&bearer_file)? + .trim() + .to_string(); + bearer = Some(SoraJwtBearerToken::new(value)?); + } + + if sentinel_file.exists() { + let value = read_to_string(&sentinel_file)? .trim() .to_string(); - return Ok(SoraCredentials { - bearer_token: bearer, - cookie, - sentinel: Some(sentinel), - }); + sentinel = Some(SoraSentinel::new(value)); } - Ok(SoraCredentials { - bearer_token: bearer, + Ok(SoraCredentialSet::initialize( cookie, - sentinel: None, - }) + bearer, + sentinel, + )) } diff --git a/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk_with_upgrade.rs b/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk_with_upgrade.rs new file mode 100644 index 0000000000..d5200c1cfd --- /dev/null +++ b/crates/desktop/tauri-artcraft/src/state/sora/read_sora_credentials_from_disk_with_upgrade.rs @@ -0,0 +1,92 @@ +use crate::state::app_dir::AppDataRoot; +use anyhow::anyhow; +use errors::AnyhowResult; +use log::{info, warn}; +use openai_sora_client::creds::sora_cookies::SoraCookies; +use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; +use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use openai_sora_client::creds::sora_sentinel::SoraSentinel; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use std::fs::{read_to_string, OpenOptions}; +use std::io::Write; + +pub async fn read_sora_credentials_from_disk_with_upgrade(app_data_root: &AppDataRoot) -> AnyhowResult { + let cookie_file = app_data_root.get_sora_cookie_file_path(); + let bearer_file = app_data_root.get_sora_bearer_token_file_path(); + let sentinel_file = app_data_root.get_sora_sentinel_file_path(); + + if !cookie_file.exists() { + return Err(anyhow!("Cookie file does not exist: {:?}", cookie_file)); + } + + let value = read_to_string(cookie_file)? + .trim() + .to_string(); + + let cookie = SoraCookies::new(value); + + let mut bearer = None; + let mut sentinel = None; + + if bearer_file.exists() { + let value = read_to_string(&bearer_file)? + .trim() + .to_string(); + bearer = Some(SoraJwtBearerToken::new(value)?); + } + + if sentinel_file.exists() { + let value = read_to_string(&sentinel_file)? + .trim() + .to_string(); + + sentinel = Some(SoraSentinel::new(value)); + } + + let mut credentials = SoraCredentialSet::initialize( + cookie, + bearer, + sentinel, + ); + + let response = maybe_upgrade_or_renew_session(&mut credentials).await; + + if let Err(err) = response { + // NB: Make this portion infallible. Don't die on setup. We can recover downstream. + warn!("Failed to upgrade or renew session: {:?}", err); + } + + if !bearer_file.exists() { + if let Some(bearer) = &credentials.jwt_bearer_token { + info!("Persisting bearer token value..."); + let value = bearer.token_str(); + + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&bearer_file)?; + + file.write_all(value.as_bytes())?; + file.flush()?; + } + } + + if !sentinel_file.exists() { + if let Some(sentinel) = &credentials.sora_sentinel { + info!("Persisting sentinel value..."); + let value = sentinel.get_sentinel(); + + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&sentinel_file)?; + + file.write_all(value.as_bytes())?; + file.flush()?; + } + } + + Ok(credentials) +} diff --git a/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_holder.rs b/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_holder.rs index 033dfc3b6a..74f68b74ca 100644 --- a/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_holder.rs +++ b/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_holder.rs @@ -1,23 +1,22 @@ -use std::ops::Deref; -use std::sync::{Arc, RwLock}; use anyhow::anyhow; use errors::AnyhowResult; -use openai_sora_client::credentials::SoraCredentials; +use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; +use std::sync::{Arc, RwLock}; /// Hold credentials for the application. #[derive(Clone)] pub struct SoraCredentialHolder { - credentials: Arc>> + credentials: Arc>>, } impl SoraCredentialHolder { pub fn new() -> Self { Self { - credentials: Arc::new(RwLock::new(None)) + credentials: Arc::new(RwLock::new(None)), } } - pub fn set_credentials(&self, credentials: &SoraCredentials) -> AnyhowResult<()> { + pub fn set_credentials(&self, credentials: &SoraCredentialSet) -> AnyhowResult<()> { match self.credentials.write() { Err(err) => Err(anyhow!("Failed to acquire write lock: {:?}", err)), Ok(mut creds) => { @@ -37,7 +36,7 @@ impl SoraCredentialHolder { } } - pub fn get_credentials(&self) -> AnyhowResult> { + pub fn get_credentials(&self) -> AnyhowResult> { match self.credentials.read() { Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)), Ok(creds) => { @@ -46,10 +45,10 @@ impl SoraCredentialHolder { } } - pub fn get_credentials_required(&self) -> AnyhowResult { + pub fn get_credentials_required(&self) -> AnyhowResult { match self.credentials.read() { Err(err) => Err(anyhow!("Failed to acquire read lock: {:?}", err)), - Ok(creds) => match creds.deref() { + Ok(creds) => match &*creds { None => Err(anyhow!("Credentials not set")), Some(creds) => Ok(creds.clone()), } diff --git a/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_manager.rs b/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_manager.rs index 63ed2b1282..5db4de9e99 100644 --- a/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_manager.rs +++ b/crates/desktop/tauri-artcraft/src/state/sora/sora_credential_manager.rs @@ -1,14 +1,16 @@ use crate::state::app_dir::AppDataRoot; use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::state::sora::sora_credential_holder::SoraCredentialHolder; +use anyhow::anyhow; use errors::AnyhowResult; use log::{error, info, warn}; -use openai_sora_client::credentials::SoraCredentials; -use openai_sora_client::sentinel_refresh::refresh_sentinel; +use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; +use openai_sora_client::creds::sora_sentinel::SoraSentinel; +use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token; use std::fs::OpenOptions; use std::io::Write; -use anyhow::anyhow; -use openai_sora_client::sentinel_refresh::generate::token::generate_token; +use openai_sora_client::creds::sora_cookies::SoraCookies; +use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken; #[derive(Clone)] pub struct SoraCredentialManager { @@ -34,30 +36,50 @@ impl SoraCredentialManager { } } - pub fn set_credentials(&self, creds: &SoraCredentials) -> AnyhowResult<()> { + pub fn set_credentials(&self, creds: &SoraCredentialSet) -> AnyhowResult<()> { self.holder.set_credentials(creds) } pub fn clear_credentials(&self) -> AnyhowResult<()> { - self.holder.clear_credentials() + self.holder.clear_credentials()?; + Ok(()) } - pub fn get_credentials(&self) -> AnyhowResult> { + pub fn get_credentials(&self) -> AnyhowResult> { self.holder.get_credentials() } - pub fn get_credentials_required(&self) -> AnyhowResult { + pub fn get_credentials_required(&self) -> AnyhowResult { self.holder.get_credentials_required() } - pub fn reset_from_disk(&self) -> AnyhowResult { + pub fn reset_from_disk(&self) -> AnyhowResult { let creds = read_sora_credentials_from_disk(&self.app_data_root)?; self.holder.set_credentials(&creds)?; Ok(creds) } + pub fn persist_all_to_disk(&self) -> AnyhowResult<()> { + let credentials = self.holder.get_credentials_required()?; + + info!("Persisting cookies to disk..."); + persist_cookies_to_disk(&credentials.cookies, &self.app_data_root)?; + + if let Some(bearer) = &credentials.jwt_bearer_token { + info!("Persisting JWT to disk..."); + persist_jwt_bearer_to_disk(bearer, &self.app_data_root)?; + } + + if let Some(sentinel) = &credentials.sora_sentinel { + info!("Persisting sentinel to disk..."); + persist_sentinel_to_disk(sentinel, &self.app_data_root)?; + } + + Ok(()) + } + /// Refresh the sentinel token from Sora's API - pub async fn call_sentinel_refresh(&self) -> AnyhowResult { + pub async fn call_sentinel_refresh(&self) -> AnyhowResult { // NB(bt,2025-04-21): Technically we don't need credentials to get a sentinel. let mut creds = self.holder.get_credentials_required()?; @@ -72,7 +94,7 @@ impl SoraCredentialManager { info!("Token obtained."); - creds.sentinel = Some(sentinel.clone()); + creds.sora_sentinel = Some(SoraSentinel::new(sentinel.clone())); self.holder.set_credentials(&creds)?; @@ -88,3 +110,48 @@ impl SoraCredentialManager { Ok(creds) } } + +fn persist_cookies_to_disk(cookies: &SoraCookies, app_data_root: &AppDataRoot) -> AnyhowResult<()> { + let filename = app_data_root.get_sora_cookie_file_path(); + + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(filename)?; + + file.write_all(cookies.as_bytes())?; + file.flush()?; + + Ok(()) +} + +fn persist_jwt_bearer_to_disk(bearer: &SoraJwtBearerToken, app_data_root: &AppDataRoot) -> AnyhowResult<()> { + let filename = app_data_root.get_sora_bearer_token_file_path(); + + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(filename)?; + + file.write_all(bearer.as_bytes())?; + file.flush()?; + + Ok(()) +} + +fn persist_sentinel_to_disk(sentinel: &SoraSentinel, app_data_root: &AppDataRoot) -> AnyhowResult<()> { + let filename = app_data_root.get_sora_sentinel_file_path(); + + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(filename)?; + + file.write_all(sentinel.as_bytes())?; + file.flush()?; + + Ok(()) +} diff --git a/crates/desktop/tauri-artcraft/src/threads/sora_session_login_thread.rs b/crates/desktop/tauri-artcraft/src/threads/sora_session_login_thread.rs index d86382173b..a68e102855 100644 --- a/crates/desktop/tauri-artcraft/src/threads/sora_session_login_thread.rs +++ b/crates/desktop/tauri-artcraft/src/threads/sora_session_login_thread.rs @@ -1,16 +1,22 @@ +use std::fs; use crate::state::app_dir::AppDataRoot; use crate::state::sora::sora_credential_holder::SoraCredentialHolder; use crate::state::sora::sora_credential_manager::SoraCredentialManager; +use crate::utils::sora::initialize_sora_jwt_bearer_token::initialize_sora_jwt_bearer_token; use crate::utils::sora_webview_cookies::get_all_sora_cookies_as_string; use anyhow::anyhow; use errors::AnyhowResult; -use log::{error, info}; +use log::{error, info, warn}; use once_cell::sync::Lazy; use reqwest::Url; -use std::fs::OpenOptions; +use std::fs::{read_to_string, OpenOptions}; use std::io::Write; +use std::ops::Sub; +use chrono::{DateTime, NaiveDateTime, TimeDelta}; use tauri::{AppHandle, Manager, Webview}; -use crate::utils::sora::initialize_sora_jwt_bearer_token::initialize_sora_jwt_bearer_token; +use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::utils::has_session_cookie::{has_session_cookie, SessionCookiePresence}; pub const LOGIN_WINDOW_NAME: &str = "login_window"; @@ -34,7 +40,11 @@ pub async fn sora_session_login_thread( loop { for (window_name, webview) in app.webviews() { if window_name == LOGIN_WINDOW_NAME { - let result = check_login_window(&webview, &app_data_root).await; + let result = check_login_window( + &webview, + &app_data_root, + &sora_creds_manager, + ).await; if let Err(err) = result { error!("Error checking login window: {:?}", err); } @@ -45,10 +55,14 @@ pub async fn sora_session_login_thread( } } -async fn check_login_window(webview: &Webview, app_data_root: &AppDataRoot) -> AnyhowResult<()> { +async fn check_login_window( + webview: &Webview, + app_data_root: &AppDataRoot, + sora_credential_manager: &SoraCredentialManager, +) -> AnyhowResult<()> { clear_browsing_data_on_test_domain(webview)?; //keep_on_task(webview)?; - extract_cookies_to_file(webview, app_data_root)?; + extract_cookies_to_file(webview, app_data_root, sora_credential_manager).await?; //initialize_sora_jwt_bearer_token(app_data_root).await?; // TODO: This only runs once. We need better management. Ok(()) } @@ -92,19 +106,67 @@ fn clear_browsing_data_on_test_domain(webview: &Webview) -> AnyhowResult<()> { } // TODO(bt,2025-04-07): Heuristic to detect when logged in. Only write when logged in. -fn extract_cookies_to_file(webview: &Webview, app_data_root: &AppDataRoot) -> AnyhowResult<()> { - let cookies = get_all_sora_cookies_as_string(webview)?; +async fn extract_cookies_to_file( + webview: &Webview, + app_data_root: &AppDataRoot, + sora_credential_manager: &SoraCredentialManager, +) -> AnyhowResult<()> { + let new_cookies = get_all_sora_cookies_as_string(webview)?.trim().to_string(); - let cookie_file = app_data_root.get_sora_cookie_file_path(); + if new_cookies.is_empty() { + return Ok(()); + } - let mut file = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(cookie_file)?; + let session_cookie_presence = has_session_cookie(&new_cookies) + .unwrap_or_else(|err| { + error!("Failed to check for session cookie: {:?}", err); + SessionCookiePresence::MaybePresent + }); - file.write_all(cookies.as_bytes())?; - file.flush()?; + info!("Session cookies are: {:?}", session_cookie_presence); + + let (should_write_cookies, should_upgrade_session) = match session_cookie_presence { + SessionCookiePresence::Present => (true, true), + SessionCookiePresence::MaybePresent => (true, false), + SessionCookiePresence::Absent => (false, false), + }; + + if !should_write_cookies { + return Ok(()); + } + + // TODO(bt): Race conditions ahead. + + sora_credential_manager.clear_credentials()?; + + let _r = fs::remove_file(app_data_root.get_sora_cookie_file_path()); + let _r = fs::remove_file(app_data_root.get_sora_bearer_token_file_path()); + let _r = fs::remove_file(app_data_root.get_sora_sentinel_file_path()); + + let mut new_credentials = + SoraCredentialSet::initialize_with_just_cookies_str(&new_cookies); + + if should_upgrade_session { + let _upgraded = maybe_upgrade_or_renew_session(&mut new_credentials).await?; + } + + sora_credential_manager.set_credentials(&new_credentials)?; + sora_credential_manager.persist_all_to_disk()?; Ok(()) } + +pub fn read_sora_cookies_from_disk(app_data_root: &AppDataRoot) -> Option { + let cookie_file = app_data_root.get_sora_cookie_file_path(); + if !cookie_file.exists() { + return None; + } + + match read_to_string(cookie_file) { + Ok(contents) => Some(contents.trim().to_string()), + Err(err) => { + warn!("Failed to read cookie file: {:?}", err); + None + } + } +} diff --git a/crates/desktop/tauri-artcraft/src/utils/sora/initialize_sora_jwt_bearer_token.rs b/crates/desktop/tauri-artcraft/src/utils/sora/initialize_sora_jwt_bearer_token.rs index cfe593a323..2953c64f57 100644 --- a/crates/desktop/tauri-artcraft/src/utils/sora/initialize_sora_jwt_bearer_token.rs +++ b/crates/desktop/tauri-artcraft/src/utils/sora/initialize_sora_jwt_bearer_token.rs @@ -2,7 +2,7 @@ use crate::state::app_dir::AppDataRoot; use anyhow::anyhow; use errors::AnyhowResult; use log::info; -use openai_sora_client::bearer::generate::generate_bearer_with_cookie; +use openai_sora_client::requests::bearer::generate::generate_bearer_with_cookie; /// Loading the JWT bearer token for the first time. pub async fn initialize_sora_jwt_bearer_token(app_data_root: &AppDataRoot) -> AnyhowResult<()> { diff --git a/crates/desktop/tauri-artcraft/tauri.conf.json b/crates/desktop/tauri-artcraft/tauri.conf.json index c5ab1444ff..3854d48f98 100644 --- a/crates/desktop/tauri-artcraft/tauri.conf.json +++ b/crates/desktop/tauri-artcraft/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2.0.0-rc", "productName": "ArtCraft", - "version": "0.1.0", + "version": "0.0.1", "identifier": "ai.artcraft.app", "build": { "frontendDist": "../../../frontend/apps/editor2d/dist", diff --git a/crates/lib/api_clients/openai_sora_client/Cargo.toml b/crates/lib/api_clients/openai_sora_client/Cargo.toml index 17536a9487..f1ad20e2f4 100644 --- a/crates/lib/api_clients/openai_sora_client/Cargo.toml +++ b/crates/lib/api_clients/openai_sora_client/Cargo.toml @@ -19,25 +19,27 @@ workspace-build-acceleration = { workspace = true } # Internal crockford = { path = "../../data/crockford" } errors = { path = "../../errors" } -subprocess_common = { path = "../../subprocess_common" } idempotency = { path = "../../idempotency" } +subprocess_common = { path = "../../subprocess_common" } # External anyhow = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true} +cookie = { workspace = true } +hex-literal = { workspace = true } log = { workspace = true } +once_cell= { workspace = true } +rand = { workspace = true } reqwest = { workspace = true, features = ["multipart", "gzip"] } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } sha3 = { workspace = true } -hex-literal = { workspace = true } -base64 = { workspace = true } +thiserror = { workspace = true } tokio = { version = "1.38.0", features = ["macros"] } toml = { workspace = true } -thiserror = { workspace = true } -chrono = { workspace = true} -rand = { workspace = true } [dev-dependencies] testing = { path = "../../testing" } -filesys = { path = "../../files/filesys" } +filesys = { path = "../../files/filesys" } \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/credentials.rs b/crates/lib/api_clients/openai_sora_client/src/credentials.rs index c9f4a68a31..b15c91080c 100644 --- a/crates/lib/api_clients/openai_sora_client/src/credentials.rs +++ b/crates/lib/api_clients/openai_sora_client/src/credentials.rs @@ -7,13 +7,18 @@ use errors::AnyhowResult; pub const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"; #[derive(Clone, Deserialize)] +#[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub struct SoraCredentials { + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub bearer_token: String, + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub cookie: String, + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub sentinel: Option, } impl SoraCredentials { + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub fn authorization_header_value(&self) -> String { match self.bearer_token.get(0..6) { Some("bearer") | Some("Bearer") => self.bearer_token.clone(), @@ -21,6 +26,7 @@ impl SoraCredentials { } } + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub fn from_env() -> AnyhowResult { let bearer = std::env::var("SORA_BEARER_TOKEN")?; let cookie = std::env::var("SORA_COOKIE")?; @@ -29,12 +35,14 @@ impl SoraCredentials { Ok(Self { bearer_token: bearer, cookie, sentinel }) } + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub fn from_toml_file>(filename: P) -> AnyhowResult { let contents = std::fs::read_to_string(filename)?; let credentials = toml::from_str(&contents)?; Ok(credentials) } + #[deprecated(note="Use `SoraCredentialSet` instead as it has auto-healing behavior and doesn't require a bearer token initially.")] pub fn add_credential_headers_to_request(&self, request: RequestBuilder) -> RequestBuilder { let bearer_header = self.authorization_header_value(); diff --git a/crates/lib/api_clients/openai_sora_client/src/creds/credential_migration.rs b/crates/lib/api_clients/openai_sora_client/src/creds/credential_migration.rs new file mode 100644 index 0000000000..d578c11571 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/creds/credential_migration.rs @@ -0,0 +1,12 @@ +use crate::credentials::SoraCredentials; +use crate::creds::sora_credential_set::SoraCredentialSet; + +pub enum CredentialMigration { + Legacy(SoraCredentials), + New(SoraCredentialSet), +} + +pub enum CredentialMigrationRef<'a> { + Legacy(&'a SoraCredentials), + New(&'a SoraCredentialSet), +} diff --git a/crates/lib/api_clients/openai_sora_client/src/creds/mod.rs b/crates/lib/api_clients/openai_sora_client/src/creds/mod.rs new file mode 100644 index 0000000000..dcd8975f61 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/creds/mod.rs @@ -0,0 +1,5 @@ +pub mod credential_migration; +pub mod sora_cookies; +pub mod sora_credential_set; +pub mod sora_jwt_bearer_token; +pub mod sora_sentinel; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/creds/sora_cookies.rs b/crates/lib/api_clients/openai_sora_client/src/creds/sora_cookies.rs new file mode 100644 index 0000000000..5ff387b645 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/creds/sora_cookies.rs @@ -0,0 +1,25 @@ +/// Cookies are the credential that are always required. +/// You can mint a JWT bearer token with just the cookies. + +#[derive(Clone)] +pub struct SoraCookies { + cookies: String, +} + +impl SoraCookies { + pub fn new(cookies: String) -> Self { + SoraCookies { cookies } + } + + pub fn as_str(&self) -> &str { + &self.cookies + } + + pub fn as_bytes(&self) -> &[u8] { + self.cookies.as_bytes() + } + + pub fn to_string(&self) -> String { + self.cookies.clone() + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/creds/sora_credential_set.rs b/crates/lib/api_clients/openai_sora_client/src/creds/sora_credential_set.rs new file mode 100644 index 0000000000..636c3cda04 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/creds/sora_credential_set.rs @@ -0,0 +1,75 @@ +use anyhow::anyhow; +use errors::AnyhowResult; +use crate::credentials::SoraCredentials; +use crate::creds::sora_cookies::SoraCookies; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use crate::creds::sora_sentinel::SoraSentinel; + +#[derive(Clone)] +pub struct SoraCredentialSet { + pub cookies: SoraCookies, + pub jwt_bearer_token: Option, + pub sora_sentinel: Option, +} + +impl SoraCredentialSet { + pub fn initialize( + cookies: SoraCookies, + bearer: Option, + sentinel: Option + ) -> Self { + Self { + cookies, + jwt_bearer_token: bearer, + sora_sentinel: sentinel, + } + } + + pub fn initialize_with_just_cookies(cookies: SoraCookies) -> Self { + SoraCredentialSet { + cookies, + jwt_bearer_token: None, + sora_sentinel: None, + } + } + + pub fn initialize_with_just_cookies_str(cookies: &str) -> Self { + Self::initialize_with_just_cookies( + SoraCookies::new(cookies.to_string()), + ) + } + + // TODO(bt,2025-04-23): Just here for migration + pub fn from_legacy_credentials(credentials: &SoraCredentials) -> AnyhowResult { + let cookies = SoraCookies::new(credentials.cookie.clone()); + let jwt_bearer_token = SoraJwtBearerToken::new(credentials.bearer_token.clone())?; + let mut sora_sentinel = None; + + if let Some(sentinel) = &credentials.sentinel { + sora_sentinel = Some(SoraSentinel::new(sentinel.clone())); + } + + Ok(SoraCredentialSet { + cookies, + jwt_bearer_token: Some(jwt_bearer_token), + sora_sentinel, + }) + } + + pub fn to_legacy_credentials(&self) -> AnyhowResult { + let bearer_token= self.jwt_bearer_token + .as_ref() + .map(|token| token.token_str().to_string()) + .ok_or_else(|| anyhow!("There is no bearer token to convert to a required legacy bearer token"))?; + + let sentinel = self.sora_sentinel + .as_ref() + .map(|sentinel| sentinel.get_sentinel().to_string()); + + Ok(SoraCredentials { + cookie: self.cookies.as_str().to_string(), + bearer_token, + sentinel, + }) + } +} \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/creds/sora_jwt_bearer_token.rs b/crates/lib/api_clients/openai_sora_client/src/creds/sora_jwt_bearer_token.rs new file mode 100644 index 0000000000..518f702d7f --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/creds/sora_jwt_bearer_token.rs @@ -0,0 +1,38 @@ +use crate::utils::lightweight_sora_jwt_parse::{lightweight_sora_jwt_parse, SoraJwtClaims}; +use errors::AnyhowResult; + +/// Sora bearer tokens are JWT tokens that can be minted with a valid cookie/session. +#[derive(Clone)] +pub struct SoraJwtBearerToken { + token: String, + jwt_claims: SoraJwtClaims, +} + +impl SoraJwtBearerToken { + pub fn new(token: String) -> AnyhowResult { + let jwt_claims = lightweight_sora_jwt_parse(&token)?; + Ok(SoraJwtBearerToken { + token, + jwt_claims, + }) + } + + pub fn token_str(&self) -> &str { + &self.token + } + + pub fn as_bytes(&self) -> &[u8] { + self.token.as_bytes() + } + + pub fn jwt_claims(&self) -> &SoraJwtClaims { + &self.jwt_claims + } + + pub fn to_authorization_header_value(&self) -> String { + match self.token.get(0..6) { + Some("bearer") | Some("Bearer") => self.token.clone(), + _ => "Bearer ".to_owned() + &self.token, + } + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/creds/sora_sentinel.rs b/crates/lib/api_clients/openai_sora_client/src/creds/sora_sentinel.rs new file mode 100644 index 0000000000..da9e492801 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/creds/sora_sentinel.rs @@ -0,0 +1,23 @@ + +/// Sora Sentinel tokens can be generated without cookies or JWTs, and are needed for some API calls. +#[derive(Clone)] +pub struct SoraSentinel { + sentinel: String, + + // TODO(bt,2025-04-23): We may want to store the user agent alongside the sentinel + // as that appears to be important for the validity of the sentinel token. +} + +impl SoraSentinel { + pub fn new(sentinel: String) -> Self { + SoraSentinel { sentinel } + } + + pub fn get_sentinel(&self) -> &str { + &self.sentinel + } + + pub fn as_bytes(&self) -> &[u8] { + self.sentinel.as_bytes() + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/lib.rs b/crates/lib/api_clients/openai_sora_client/src/lib.rs index 56e662dbda..f3c2e26b58 100644 --- a/crates/lib/api_clients/openai_sora_client/src/lib.rs +++ b/crates/lib/api_clients/openai_sora_client/src/lib.rs @@ -1,7 +1,6 @@ -pub mod bearer; pub mod credentials; -pub mod image_gen; -pub mod job; -pub mod upload; -pub mod sentinel_refresh; -mod sora_error; \ No newline at end of file +pub mod creds; +pub mod recipes; +pub mod requests; +pub mod sora_error; +pub mod utils; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs new file mode 100644 index 0000000000..ee68c0e638 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs @@ -0,0 +1,139 @@ +use anyhow::anyhow; +use log::{info, warn}; +use errors::AnyhowResult; +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use crate::creds::sora_sentinel::SoraSentinel; +use crate::requests::bearer::generate::generate_bearer_with_cookie; +use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; +use crate::requests::image_gen::image_gen_http_request; +use crate::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use crate::requests::sentinel_refresh::generate::token::generate_token; +use crate::sora_error::SoraError; + +pub struct ImageRemixAutoRenewRequest<'a> { + pub prompt: String, + pub num_images: NumImages, + pub image_size: ImageSize, + pub sora_media_tokens: Vec, + pub credentials: &'a SoraCredentialSet, +} + +/// Image request with retry and session auto-renewal. +pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewRequest<'_>) -> Result<(SoraImageGenResponse, Option), SoraError> { + let result = sora_image_gen_remix(SoraImageGenRemixRequest { + prompt: request.prompt.clone(), // NB: Clone because used again + num_images: request.num_images, + image_size: request.image_size, + sora_media_tokens: request.sora_media_tokens.clone(), // NB: Clone because used again + credentials: CredentialMigrationRef::New(request.credentials), + }).await; + + let err = match result { + Ok(response) => return Ok((response, None)), + Err(err) => err, + }; + + warn!("Image generation failed: {:?}", err); + + let mut refresh_jwt = false; + let mut refresh_sentinel = false; + + match err { + // We'll fail these requests... + image_gen_http_request::SoraError::TooManyConcurrentTasks(_) => { + return Err(SoraError::TooManyConcurrentTasks); + } + image_gen_http_request::SoraError::GenericError(err) => { + return Err(SoraError::OtherBadStatus(anyhow!("image remix failed with GenericError: {:?}", err))) + } + image_gen_http_request::SoraError::NetworkError(err) => { + // TODO: The underlying type should be a reqwest::Error. + return Err(SoraError::OtherBadStatus(anyhow!("network error: {:?}", err))) + } + + // We'll retry these requests... + + image_gen_http_request::SoraError::SentinelBlock(err) => { + warn!("Image generation failed due to sentinel block error: {:?}", err); + refresh_sentinel = true; + } + image_gen_http_request::SoraError::TokenExpired(err) => { + warn!("Image generation failed due to token expired error: {:?}", err); + refresh_sentinel = true; // TODO: Not sure what this error is, actually. + refresh_jwt = true; + } + image_gen_http_request::SoraError::InvalidJwt(err) => { + warn!("Image generation failed due to invalid jwt error: {:?}", err); + refresh_jwt = true; + } + } + + let mut new_bearer = None; + + if refresh_jwt { + info!("Generating new JWT bearer token..."); + let cookies = request.credentials.cookies.as_str(); + let response = generate_bearer_with_cookie(cookies).await; + match response { + Err(err) => { + return Err(SoraError::OtherBadStatus(anyhow!("failed to generate new JWT bearer token: {:?}", err))) + } + Ok(bearer) => { + match SoraJwtBearerToken::new(bearer) { + Err(err) => { + return Err(SoraError::OtherBadStatus(anyhow!("Failed to parse new JWT bearer token: {:?}", err))); + } + Ok(bearer) => new_bearer = Some(bearer), + } + }, + } + } + + let mut new_sentinel = None; + + if refresh_sentinel { + info!("Generating new sentinel..."); + let response = generate_token().await; + match response { + Err(err) => { + return Err(SoraError::OtherBadStatus(anyhow!("failed to generate new sentinel: {:?}", err))) + } + Ok(sentinel) => new_sentinel = Some(SoraSentinel::new(sentinel)), + } + } + + let mut new_creds = request.credentials.clone(); + + if let Some(bearer) = new_bearer { + new_creds.jwt_bearer_token = Some(bearer); + } + + if let Some(sentinel) = new_sentinel { + new_creds.sora_sentinel = Some(sentinel); + } + + // Now try again... + + let result = sora_image_gen_remix(SoraImageGenRemixRequest { + prompt: request.prompt, + num_images: request.num_images, + image_size: request.image_size, + sora_media_tokens: request.sora_media_tokens, + credentials: CredentialMigrationRef::New(&new_creds), + }).await; + + match result { + Ok(response) => Ok((response, Some(new_creds))), + Err(err) => match err { + image_gen_http_request::SoraError::TooManyConcurrentTasks(err) => { + Err(SoraError::TooManyConcurrentTasks) + } + _ => { + warn!("Image remix failed again: {:?}", err); + Err(SoraError::OtherBadStatus(anyhow!("image remix failed again: {:?}", err))) + } + } + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs new file mode 100644 index 0000000000..a356a277be --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs @@ -0,0 +1,56 @@ +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use crate::creds::sora_sentinel::SoraSentinel; +use crate::recipes::maybe_refresh_credentials_on_sora_error::maybe_refresh_credentials_on_sora_error; +use crate::requests::bearer::generate::generate_bearer_with_cookie; +use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; +use crate::requests::image_gen::image_gen_http_request; +use crate::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use crate::requests::sentinel_refresh::generate::token::generate_token; +use crate::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; +use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; +use crate::sora_error::SoraError; +use anyhow::anyhow; +use errors::AnyhowResult; +use log::{error, info, warn}; + +pub struct ImageUploadFromBytesAutoRenewRequest<'a> { + pub file_bytes: Vec, + pub filename: String, + pub mime_type: &'a str, + pub credentials: &'a SoraCredentialSet, +} + +/// Image upload with retry and session auto-renewal. +pub async fn image_upload_from_bytes_with_session_auto_renew( + request: ImageUploadFromBytesAutoRenewRequest<'_> +) -> Result<(SoraMediaUploadResponse, Option), SoraError> { + + let result = sora_media_upload_from_bytes( + request.file_bytes.clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :( + request.filename.clone(), // FIXME: Same + CredentialMigrationRef::New(request.credentials), + ).await; + + let err = match result { + Ok(response) => return Ok((response, None)), + Err(err) => err, + }; + + warn!("Image upload failed: {:?}", err); + + let new_creds = maybe_refresh_credentials_on_sora_error(request.credentials, err).await?; + + // Now try again... + + info!("Retrying image upload with new credentials..."); + + let result = sora_media_upload_from_bytes( + request.file_bytes, + request.filename, + CredentialMigrationRef::New(&new_creds), + ).await?; + + Ok((result, Some(new_creds))) +} diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs new file mode 100644 index 0000000000..79b31bff0b --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs @@ -0,0 +1,43 @@ +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::recipes::maybe_refresh_credentials_on_sora_error::maybe_refresh_credentials_on_sora_error; +use crate::requests::upload::upload_media_from_file::sora_media_upload_from_file; +use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; +use crate::sora_error::SoraError; +use log::{info, warn}; +use std::path::Path; + +pub struct ImageUploadFromFileAutoRenewRequest<'a, P: AsRef> { + pub file_path: P, + pub credentials: &'a SoraCredentialSet, +} + +/// Image upload with retry and session auto-renewal. +pub async fn image_upload_from_file_with_session_auto_renew>( + request: ImageUploadFromFileAutoRenewRequest<'_, P> +) -> Result<(SoraMediaUploadResponse, Option), SoraError> { + + let result = sora_media_upload_from_file( + request.file_path.as_ref().clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :( + CredentialMigrationRef::New(request.credentials), + ).await; + + let err = match result { + Ok(response) => return Ok((response, None)), + Err(err) => err, + }; + + warn!("Image upload failed: {:?}", err); + + let new_creds = maybe_refresh_credentials_on_sora_error(request.credentials, err).await?; + + info!("Retrying image upload with new credentials..."); + + let result = sora_media_upload_from_file( + request.file_path, + CredentialMigrationRef::New(&new_creds), + ).await?; + + Ok((result, Some(new_creds))) +} + diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/maybe_refresh_credentials_on_sora_error.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/maybe_refresh_credentials_on_sora_error.rs new file mode 100644 index 0000000000..82d893956a --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/maybe_refresh_credentials_on_sora_error.rs @@ -0,0 +1,49 @@ +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use crate::requests::bearer::generate::generate_bearer_with_cookie; +use crate::sora_error::SoraError; +use anyhow::anyhow; +use log::{error, info, warn}; + +// TODO: Make other APIs work with this. + +pub async fn maybe_refresh_credentials_on_sora_error(creds: &SoraCredentialSet, error: SoraError) -> Result { + + match error { + SoraError::NoBearerTokenAvailable => { + warn!("Previous request failed due to missing bearer token."); + } + SoraError::UnauthorizedCookieOrBearerExpired => { + warn!("Previous request failed due to invalid bearer token."); + } + _ => { + error!("Previous request failed with error: {:?}", error); + return Err(SoraError::OtherBadStatus(anyhow!("previous request failed with error: {:?}", error))) + } + } + + info!("Generating new JWT bearer token..."); + + let cookies = creds.cookies.as_str(); + let response = generate_bearer_with_cookie(cookies).await; + + let new_bearer = match response { + Err(err) => { + return Err(SoraError::OtherBadStatus(anyhow!("failed to generate new JWT bearer token: {:?}", err))) + } + Ok(bearer) => { + match SoraJwtBearerToken::new(bearer) { + Err(err) => { + return Err(SoraError::OtherBadStatus(anyhow!("failed to parse new JWT bearer token: {:?}", err))); + } + Ok(bearer) => bearer, + } + } + }; + + let mut new_creds = creds.clone(); + + new_creds.jwt_bearer_token = Some(new_bearer); + + Ok(new_creds) +} diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs new file mode 100644 index 0000000000..98b2f284f0 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs @@ -0,0 +1,55 @@ +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use crate::creds::sora_sentinel::SoraSentinel; +use crate::requests::bearer::generate::generate_bearer_with_cookie; +use crate::requests::sentinel_refresh::generate::token::generate_token; +use chrono::{DateTime, TimeDelta, Utc}; +use errors::AnyhowResult; +use log::info; +use std::ops::Sub; + +const EXPIRATION_DEADLINE : TimeDelta = TimeDelta::hours(12); + +/// Call this at session startup to generate the tokens for the first time. +/// Call this periodically to refresh the JWT as it nears its expiration date. +/// This will not refresh the sentinel token once it's been generated - we can use other flows to accomplish that. +pub async fn maybe_upgrade_or_renew_session(sora_credentials: &mut SoraCredentialSet) -> AnyhowResult { + let mut refresh_jwt = !sora_credentials.jwt_bearer_token.is_some(); + let refresh_sentinel = !sora_credentials.sora_sentinel.is_some(); + + if let Some(jwt) = &sora_credentials.jwt_bearer_token { + let now = Utc::now(); + let refresh_deadline = now.sub(EXPIRATION_DEADLINE); + if jwt.jwt_claims().expiration.lt(&now) { + info!("JWT expiration time has elapsed."); + refresh_jwt = true; + } else if jwt.jwt_claims().expiration.lt(&refresh_deadline) { + info!("JWT expiration time is under the expiration deadline."); + refresh_jwt = true; + } + } + + let mut credential_updated = false; + + if refresh_jwt { + let cookies = sora_credentials.cookies.as_str(); + + info!("Generating new JWT bearer token..."); + let token = generate_bearer_with_cookie(cookies).await?; + + info!("Parsing JWT bearer token."); + let token = SoraJwtBearerToken::new(token)?; + sora_credentials.jwt_bearer_token = Some(token); + credential_updated = true; + } + + if refresh_sentinel { + info!("Generating new sentinel..."); + let token = generate_token().await?; + let token = SoraSentinel::new(token); + sora_credentials.sora_sentinel = Some(token); + credential_updated = true; + } + + Ok(credential_updated) +} diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/mod.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/mod.rs new file mode 100644 index 0000000000..db5d9f692d --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/mod.rs @@ -0,0 +1,6 @@ +pub mod image_remix_with_session_auto_renew; +pub mod image_upload_from_bytes_with_session_auto_renew; +pub mod image_upload_from_file_with_session_auto_renew; +pub mod maybe_refresh_credentials_on_sora_error; +pub mod maybe_upgrade_or_renew_session; +pub mod wait_for_image_gen_status; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs new file mode 100644 index 0000000000..79a9a71d89 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs @@ -0,0 +1,51 @@ +use crate::credentials::SoraCredentials; +use crate::requests::image_gen::image_gen_status::TaskStatus; +use crate::requests::image_gen::image_gen_status::{get_image_gen_status, StatusRequest, TaskResponse, VideoGenStatusResponse}; +use errors::AnyhowResult; + +pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCredentials, retry_limit: Option) -> AnyhowResult { + let status_request = StatusRequest { + limit: None, + before: None, + // before: Some(task_id.clone()), + }; + let retry_limit = retry_limit.unwrap_or(10); + + for _ in 0..retry_limit { + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + + let status_response = get_image_gen_status(&status_request, credentials).await?; + + match find_task_response_by_id(&status_response, task_id.clone()) { + Some(task_response) => match TaskStatus::from_str(&task_response.status) { + TaskStatus::Succeeded => { + return Ok(task_response.clone()); + }, + TaskStatus::Failed => { + return Err(anyhow::anyhow!("Task failed")); + }, + TaskStatus::Unknown(status) => { + println!("Unknown task status: {}", status); + continue; + }, + TaskStatus::Queued => { + println!("Task is queued"); + continue; + }, + TaskStatus::Running => { + println!("Task is running"); + continue; + }, + }, + None => { + return Err(anyhow::anyhow!("Task not found")); + }, + } + } + + Err(anyhow::anyhow!("Task not found")) +} + +fn find_task_response_by_id(status_response: &VideoGenStatusResponse, task_id: String) -> Option<&TaskResponse> { + status_response.task_responses.iter().find(|task| task.id == task_id) +} diff --git a/crates/lib/api_clients/openai_sora_client/src/bearer/generate.rs b/crates/lib/api_clients/openai_sora_client/src/requests/bearer/generate.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/bearer/generate.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/bearer/generate.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/bearer/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/bearer/mod.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/bearer/mod.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/bearer/mod.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/common.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/common.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/common.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/common.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs similarity index 75% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs index d0a1d39750..8ef4c72abb 100644 --- a/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_http_request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs @@ -1,5 +1,5 @@ +use crate::creds::credential_migration::CredentialMigrationRef; use serde_derive::{Deserialize, Serialize}; -use crate::credentials::SoraCredentials; use thiserror::Error; const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen"; @@ -28,13 +28,13 @@ pub enum SoraError { NetworkError(String), } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] pub (crate) enum VideoGenType { ImageGen, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] pub (crate) enum OperationType { /// Simple prompt without reference images @@ -43,13 +43,13 @@ pub (crate) enum OperationType { Remix, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] pub (crate) enum InpaintItemType { Image, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] pub (crate) struct InpaintItem { pub r#type: InpaintItemType, @@ -69,7 +69,7 @@ pub (crate) struct InpaintItem { pub crop_bounds: Option, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] pub (crate) struct RawSoraImageGenRequest { /// eg. "image_gen" @@ -90,7 +90,7 @@ pub (crate) struct RawSoraImageGenRequest { pub inpaint_items: Vec, } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub (crate) struct RawSoraResponse { /// eg. "task_01jqsz9dsae9tvjygf1abrv3xf" @@ -146,20 +146,41 @@ impl std::fmt::Display for RawSoraErrorInner { } /// Don't expose the internal request implementation as there are only a few "correct" ways to call the API. -pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: &SoraCredentials) -> Result { +pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: CredentialMigrationRef<'_>) -> Result { let client = reqwest::Client::new(); - let sentinel = match credentials.sentinel.as_deref() { - None => return Err(SoraError::GenericError("Calls to image gen require a sentinel in the credentials payload.".to_string())), - Some(sentinel) => sentinel, - }; + let mut cookie; + let mut authorization_header; + let mut sentinel; + + match credentials { + CredentialMigrationRef::Legacy(creds) => { + cookie = creds.cookie.clone(); + authorization_header = creds.authorization_header_value(); + // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. + sentinel = creds.sentinel.as_ref() + .map(|sentinel| sentinel.to_string()) + .ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?; + } + CredentialMigrationRef::New(creds) => { + cookie = creds.cookies.to_string(); + // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. + authorization_header = creds.jwt_bearer_token.as_ref() + .ok_or(SoraError::InvalidJwt("JWT bearer is required for image generation".to_string()))? + .to_authorization_header_value(); + // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. + sentinel = creds.sora_sentinel.as_ref() + .map(|sentinel| sentinel.get_sentinel().to_string()) + .ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?; + } + } let http_request = client.post(SORA_IMAGE_GEN_URL) .header("User-Agent", USER_AGENT) - .header("Cookie", &credentials.cookie) - .header("Authorization", credentials.authorization_header_value()) + .header("Cookie", &cookie) + .header("Authorization", &authorization_header) .header("Content-Type", "application/json") - .header("OpenAI-Sentinel-Token", sentinel); + .header("OpenAI-Sentinel-Token", &sentinel); let http_request = http_request.json(&sora_request).build() .map_err(|e| SoraError::NetworkError(e.to_string()))?; diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_status.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs similarity index 85% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_status.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs index d4ce76092b..f1012da827 100644 --- a/crates/lib/api_clients/openai_sora_client/src/image_gen/image_gen_status.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs @@ -220,47 +220,6 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: & Ok(response) } -pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCredentials, retry_limit: Option) -> AnyhowResult { - let status_request = StatusRequest { - limit: None, - before: None, - // before: Some(task_id.clone()), - }; - let retry_limit = retry_limit.unwrap_or(10); - - for _ in 0..retry_limit { - tokio::time::sleep(std::time::Duration::from_secs(10)).await; - let status_response = get_image_gen_status(&status_request, credentials).await?; - match find_task_response_by_id(&status_response, task_id.clone()) { - Some(task_response) => match TaskStatus::from_str(&task_response.status) { - TaskStatus::Succeeded => { - return Ok(task_response.clone()); - }, - TaskStatus::Failed => { - return Err(anyhow::anyhow!("Task failed")); - }, - TaskStatus::Unknown(status) => { - println!("Unknown task status: {}", status); - continue; - }, - TaskStatus::Queued => { - println!("Task is queued"); - continue; - }, - TaskStatus::Running => { - println!("Task is running"); - continue; - }, - }, - None => { - return Err(anyhow::anyhow!("Task not found")); - }, - } - } - - Err(anyhow::anyhow!("Task not found")) -} - pub async fn save_generations_to_dir(generations: &[Generation], dir: &str) -> AnyhowResult<()> { let dir = Path::new(dir); if !dir.exists() { @@ -288,14 +247,11 @@ pub async fn save_generations_to_dir(generations: &[Generation], dir: &str) -> A Ok(()) } -fn find_task_response_by_id(status_response: &VideoGenStatusResponse, task_id: String) -> Option<&TaskResponse> { - status_response.task_responses.iter().find(|task| task.id == task_id) -} - #[cfg(test)] mod tests { use crate::credentials::SoraCredentials; - use crate::image_gen::image_gen_status::{get_image_gen_status, save_generations_to_dir, wait_for_image_gen_status, StatusRequest, VideoGenStatusResponse}; + use crate::recipes::wait_for_image_gen_status::wait_for_image_gen_status; + use crate::requests::image_gen::image_gen_status::{get_image_gen_status, save_generations_to_dir, StatusRequest, VideoGenStatusResponse}; use errors::AnyhowResult; use std::fs::read_to_string; use testing::test_file_path::test_file_path; diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs similarity index 52% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/mod.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs index 525b0fbf62..f0247756fb 100644 --- a/crates/lib/api_clients/openai_sora_client/src/image_gen/mod.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs @@ -4,4 +4,3 @@ pub mod common; pub mod sora_image_gen_remix; pub mod sora_image_gen_simple; pub mod image_gen_status; -pub use image_gen_status::{get_image_gen_status, save_generations_to_dir, wait_for_image_gen_status, Generation, StatusRequest, TaskResponse, VideoGenStatusResponse}; diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/sora_image_gen_remix.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs similarity index 74% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/sora_image_gen_remix.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs index 7db1564ed1..e6019d219c 100644 --- a/crates/lib/api_clients/openai_sora_client/src/image_gen/sora_image_gen_remix.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs @@ -1,14 +1,14 @@ -use errors::AnyhowResult; -use crate::credentials::SoraCredentials; -use crate::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; -use crate::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraError, VideoGenType}; +use log::info; +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; +use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraError, VideoGenType}; pub struct SoraImageGenRemixRequest<'a> { pub prompt: String, pub num_images: NumImages, pub image_size: ImageSize, pub sora_media_tokens: Vec, - pub credentials: &'a SoraCredentials, + pub credentials: CredentialMigrationRef<'a>, } /// The "remix" commands let you supply additional images as context. @@ -36,7 +36,10 @@ pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Resu }).collect(), }; - let result = image_gen_http_request(args, &request.credentials).await?; + // TODO: Replace this with debug level logging in the future. + info!("Sending remix request: {:?}", args); + + let result = image_gen_http_request(args, request.credentials).await?; Ok(SoraImageGenResponse { task_id: result.id, @@ -45,12 +48,13 @@ pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Resu #[cfg(test)] mod tests { - use std::fs::read_to_string; - use errors::AnyhowResult; - use testing::test_file_path::test_file_path; use crate::credentials::SoraCredentials; - use crate::image_gen::common::{ImageSize, NumImages}; - use crate::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; + use crate::creds::credential_migration::CredentialMigrationRef; + use crate::requests::image_gen::common::{ImageSize, NumImages}; + use crate::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. #[tokio::test] @@ -75,7 +79,7 @@ mod tests { num_images: NumImages::One, image_size: ImageSize::Square, sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()], - credentials: &creds, + credentials: CredentialMigrationRef::Legacy(&creds), }).await?; println!("task_id: {}", response.task_id); diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/sora_image_gen_simple.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs similarity index 76% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/sora_image_gen_simple.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs index c2f2131dc8..df41c6848a 100644 --- a/crates/lib/api_clients/openai_sora_client/src/image_gen/sora_image_gen_simple.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs @@ -1,13 +1,14 @@ -use errors::AnyhowResult; use crate::credentials::SoraCredentials; -use crate::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; -use crate::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType}; +use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; +use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType}; +use errors::AnyhowResult; +use crate::creds::credential_migration::CredentialMigrationRef; pub struct SoraImageGenSimpleRequest<'a> { pub prompt: String, pub num_images: NumImages, pub image_size: ImageSize, - pub credentials: &'a SoraCredentials, + pub credentials: CredentialMigrationRef<'a>, } pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult { @@ -32,12 +33,13 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho #[cfg(test)] mod tests { - use std::fs::read_to_string; - use errors::AnyhowResult; - use testing::test_file_path::test_file_path; use crate::credentials::SoraCredentials; - use crate::image_gen::common::{ImageSize, NumImages}; - use crate::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest}; + use crate::requests::image_gen::common::{ImageSize, NumImages}; + use crate::requests::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; + use crate::creds::credential_migration::CredentialMigrationRef; #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. #[tokio::test] @@ -61,7 +63,7 @@ mod tests { prompt: "A pirate and a ninja fight in a battle inside a UFO. Fully photo realistic, lifelike, lens flare".to_string(), num_images: NumImages::One, image_size: ImageSize::Square, - credentials: &creds, + credentials: CredentialMigrationRef::Legacy(&creds), }).await?; println!("task_id: {}", response.task_id); diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/test_status_response.json b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/test_status_response.json similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/test_status_response.json rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/test_status_response.json diff --git a/crates/lib/api_clients/openai_sora_client/src/image_gen/test_status_response_long.json b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/test_status_response_long.json similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/image_gen/test_status_response_long.json rename to crates/lib/api_clients/openai_sora_client/src/requests/image_gen/test_status_response_long.json diff --git a/crates/lib/api_clients/openai_sora_client/src/job/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/job/mod.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/job/mod.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/job/mod.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/job/sora_job_status.rs b/crates/lib/api_clients/openai_sora_client/src/requests/job/sora_job_status.rs similarity index 57% rename from crates/lib/api_clients/openai_sora_client/src/job/sora_job_status.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/job/sora_job_status.rs index c33f9541f9..15e95ca335 100644 --- a/crates/lib/api_clients/openai_sora_client/src/job/sora_job_status.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/job/sora_job_status.rs @@ -1,29 +1,45 @@ +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::sora_error::SoraError; + const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"; // https://sora.com/backend/notif?limit=100&before=task_01jqwwrkvgeqp8jsf5mqk1jceh -use crate::credentials::SoraCredentials; - pub async fn sora_job_status( task_id: &str, - credentials: &SoraCredentials, -) -> anyhow::Result<()> { + credentials: CredentialMigrationRef<'_>, +) -> Result<(), SoraError> { let url = format!("https://sora.com/backend/notif?limit=100&before={}", task_id); + let mut cookie; + let mut authorization_header; + + match credentials { + CredentialMigrationRef::Legacy(creds) => { + cookie = creds.cookie.clone(); + authorization_header = creds.authorization_header_value(); + } + CredentialMigrationRef::New(creds) => { + cookie = creds.cookies.to_string(); + // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. + authorization_header = creds.jwt_bearer_token.as_ref() + .ok_or(SoraError::NoBearerTokenAvailable)? + .to_authorization_header_value(); + } + } + let client = reqwest::Client::new(); let request = client .get(url) - .header("Authorization", &credentials.bearer_token) + .header("Authorization", &authorization_header) .header("User-Agent", USER_AGENT) - .header("Cookie", &credentials.cookie); + .header("Cookie", &cookie); let response = request.send().await?; let json_response = &response.text().await?; - println!(" >>> response = {:?}", json_response); - //let response = serde_json::from_str(json_response)?; Ok(()) @@ -31,11 +47,12 @@ pub async fn sora_job_status( #[cfg(test)] mod tests { - use std::fs::read_to_string; - use errors::AnyhowResult; - use testing::test_file_path::test_file_path; use crate::credentials::SoraCredentials; - use crate::job::sora_job_status::sora_job_status; + use crate::creds::credential_migration::CredentialMigrationRef; + use crate::requests::job::sora_job_status::sora_job_status; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; #[ignore] #[tokio::test] @@ -57,7 +74,7 @@ mod tests { let task_id = "task_01jqwwrkvgeqp8jsf5mqk1jceh"; - let result = sora_job_status(task_id, &creds).await?; + let result = sora_job_status(task_id, CredentialMigrationRef::Legacy(&creds)).await?; Ok(()) } diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/mod.rs new file mode 100644 index 0000000000..29da77e446 --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/requests/mod.rs @@ -0,0 +1,5 @@ +pub mod bearer; +pub mod image_gen; +pub mod job; +pub mod sentinel_refresh; +pub mod upload; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/config.rs b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/config.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/config.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/config.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/mod.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/mod.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/mod.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/request.rs similarity index 97% rename from crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/request.rs index aabf9be1fd..9393443f15 100644 --- a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/request.rs @@ -1,10 +1,10 @@ -use crate::sentinel_refresh::generate::config::{get_random_core_count, get_random_document_key, get_random_navigator_key, get_random_window_key}; +use crate::requests::sentinel_refresh::generate::config::{get_random_core_count, get_random_document_key, get_random_navigator_key, get_random_window_key}; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; use chrono::FixedOffset; use chrono::Utc; use idempotency::uuid::generate_random_uuid; use rand::Rng; -use base64::prelude::BASE64_STANDARD; -use base64::Engine; use sha3::{Digest, Sha3_512}; // [ diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/token.rs b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/token.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/generate/token.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/generate/token.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/mod.rs new file mode 100644 index 0000000000..118c66d72d --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/requests/sentinel_refresh/mod.rs @@ -0,0 +1 @@ +pub mod generate; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/upload/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/mod.rs similarity index 100% rename from crates/lib/api_clients/openai_sora_client/src/upload/mod.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/upload/mod.rs diff --git a/crates/lib/api_clients/openai_sora_client/src/upload/upload_media_from_bytes.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs similarity index 65% rename from crates/lib/api_clients/openai_sora_client/src/upload/upload_media_from_bytes.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs index 1877dc4892..1902b9ada9 100644 --- a/crates/lib/api_clients/openai_sora_client/src/upload/upload_media_from_bytes.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs @@ -1,16 +1,11 @@ -use std::path::{Path, PathBuf}; -use errors::AnyhowResult; -use crate::credentials::SoraCredentials; -use crate::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; - -pub struct SoraMediaUploadRequest<'a> { - pub file_path: String, - pub credentials: &'a SoraCredentials, -} +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; +use crate::sora_error::SoraError; +use std::path::PathBuf; /// Upload bytes. /// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference. -pub async fn sora_media_upload_from_bytes(bytes: Vec, file_name: String, creds: &SoraCredentials) -> AnyhowResult { +pub async fn sora_media_upload_from_bytes(bytes: Vec, file_name: String, creds: CredentialMigrationRef<'_>) -> Result { let file_path = PathBuf::from(&file_name); // TODO: Read file magic bytes first, then fall back to this. diff --git a/crates/lib/api_clients/openai_sora_client/src/upload/upload_media_from_file.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs similarity index 81% rename from crates/lib/api_clients/openai_sora_client/src/upload/upload_media_from_file.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs index d7f00d1556..dc023285c1 100644 --- a/crates/lib/api_clients/openai_sora_client/src/upload/upload_media_from_file.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs @@ -1,20 +1,15 @@ +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; +use crate::sora_error::SoraError; use std::path::Path; use tokio::fs::File; use tokio::io::AsyncReadExt; -use errors::AnyhowResult; -use crate::credentials::SoraCredentials; -use crate::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; /// Try to prevent buffer reallocations. /// There's a better way to implement this. const INITIAL_BUFFER_SIZE : usize = 1024*1024; -pub struct SoraMediaUploadRequest<'a> { - pub file_path: String, - pub credentials: &'a SoraCredentials, -} - -pub async fn sora_media_upload_from_file>(file_path: P, creds: &SoraCredentials) -> AnyhowResult { +pub async fn sora_media_upload_from_file>(file_path: P, creds: CredentialMigrationRef<'_>) -> Result { let mut file = File::open(&file_path).await?; let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE); file.read_to_end(&mut buffer).await?; @@ -41,11 +36,12 @@ pub async fn sora_media_upload_from_file>(file_path: P, creds: &S #[cfg(test)] mod tests { - use std::fs::read_to_string; - use errors::AnyhowResult; - use testing::test_file_path::test_file_path; use crate::credentials::SoraCredentials; - use crate::upload::upload_media_from_file::sora_media_upload_from_file; + use crate::creds::credential_migration::CredentialMigrationRef; + use crate::requests::upload::upload_media_from_file::sora_media_upload_from_file; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. #[tokio::test] @@ -66,7 +62,7 @@ mod tests { let response = sora_media_upload_from_file( image_path, - &creds, + CredentialMigrationRef::Legacy(&creds), ).await?; println!("media: {:?}", response); diff --git a/crates/lib/api_clients/openai_sora_client/src/upload/upload_media_http_request.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs similarity index 84% rename from crates/lib/api_clients/openai_sora_client/src/upload/upload_media_http_request.rs rename to crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs index bdb0decbe0..586c1b88ea 100644 --- a/crates/lib/api_clients/openai_sora_client/src/upload/upload_media_http_request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs @@ -1,9 +1,10 @@ +use log::info; use crate::credentials::{SoraCredentials, USER_AGENT}; -use errors::AnyhowResult; +use crate::creds::credential_migration::CredentialMigrationRef; +use crate::sora_error::SoraError; use reqwest::multipart::{Form, Part}; use reqwest::Client; use serde::Deserialize; -use crate::sora_error::SoraError; const SORA_UPLOAD_MEDIA_URL: &str = "https://sora.com/backend/uploads"; @@ -66,7 +67,7 @@ pub (crate) async fn upload_media_http_request( file_bytes: Vec, filename: String, mime_type: &str, - credentials: &SoraCredentials, + credentials: CredentialMigrationRef<'_>, ) -> Result { // Create multipart form @@ -76,18 +77,36 @@ pub (crate) async fn upload_media_http_request( let form = Form::new().part("file", part); + let cookie; + let auth_header; + + match credentials { + CredentialMigrationRef::Legacy(creds) => { + cookie = creds.cookie.clone(); + auth_header = creds.authorization_header_value(); + } + CredentialMigrationRef::New(creds) => { + cookie = creds.cookies.to_string(); + auth_header = creds.jwt_bearer_token + .as_ref() + .ok_or(SoraError::NoBearerTokenAvailable)? + .to_authorization_header_value(); + } + } + // Make API request let client = Client::new(); let request_builder = client.post(SORA_UPLOAD_MEDIA_URL) .multipart(form) .header("User-Agent", USER_AGENT) - .header("Cookie", &credentials.cookie) - .header("Authorization", credentials.authorization_header_value()); + .header("Cookie", &cookie) + .header("Authorization", &auth_header); let response = request_builder.send().await?; // Check response status if !response.status().is_success() { + info!("Error uploading image: {:?}", response.status()); let error = classify_error(response).await; return Err(error); } diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs b/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs deleted file mode 100644 index 869169304e..0000000000 --- a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub use refresh::refresh_sentinel; -pub mod refresh; - -pub mod generate; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/sora_error.rs b/crates/lib/api_clients/openai_sora_client/src/sora_error.rs index 7266b86595..78fa2e56e7 100644 --- a/crates/lib/api_clients/openai_sora_client/src/sora_error.rs +++ b/crates/lib/api_clients/openai_sora_client/src/sora_error.rs @@ -1,8 +1,16 @@ use std::error::Error; use std::fmt::{Display, Formatter}; +use std::io; +/// The single error type this crate should surface to all callers. #[derive(Debug)] pub enum SoraError { + /// We haven't received a bearer token yet (this is our application error) + NoBearerTokenAvailable, + + /// We're sending too many tasks to Sora. + TooManyConcurrentTasks, + /// Unauthorized, cookie and/or bearer token expired. We'll need to ask for a refreshed login. /// /// Example message, e.g. from the upload endpoint: @@ -21,6 +29,12 @@ pub enum SoraError { /// Reqwest Error ReqwestError(reqwest::Error), + + /// std::io Error that arises from our end, eg. reading from the filesystem. + IoError(io::Error), + + /// anyhow::Error arises from our end + AnyhowError(anyhow::Error), } impl Error for SoraError {} @@ -28,21 +42,45 @@ impl Error for SoraError {} impl Display for SoraError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - SoraError::UnauthorizedCookieOrBearerExpired => { + Self::NoBearerTokenAvailable => { + write!(f, "Unauthorized: no bearer token set") + } + Self::TooManyConcurrentTasks => { + write!(f, "Too many concurrent tasks. Please wait.") + } + Self::UnauthorizedCookieOrBearerExpired => { write!(f, "Unauthorized: cookie and/or bearer token expired") } - SoraError::OtherBadStatus(err) => { + Self::OtherBadStatus(err) => { write!(f, "Other error: {}", err) } - SoraError::ReqwestError(err) => { + Self::ReqwestError(err) => { write!(f, "Reqwest error: {}", err) } + Self::IoError(err) => { + write!(f, "IO error: {}", err) + } + Self::AnyhowError(err) => { + write!(f, "Anyhow error: {}", err) + } } } } impl From for SoraError { fn from(err: reqwest::Error) -> SoraError { - SoraError::ReqwestError(err) + Self::ReqwestError(err) + } +} + +impl From for SoraError { + fn from(err: io::Error) -> Self { + Self::IoError(err) + } +} + +impl From for SoraError { + fn from(err: anyhow::Error) -> Self { + Self::AnyhowError(err) } } diff --git a/crates/lib/api_clients/openai_sora_client/src/utils/has_session_cookie.rs b/crates/lib/api_clients/openai_sora_client/src/utils/has_session_cookie.rs new file mode 100644 index 0000000000..f5a9c3ab2e --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/utils/has_session_cookie.rs @@ -0,0 +1,123 @@ +use std::collections::HashSet; +use std::iter::Map; +use cookie::Cookie; +use once_cell::sync::Lazy; +use errors::AnyhowResult; + +pub const SESSION_COOKIE_NAME : &str = "__Secure-authjs.session-token"; + +/// Cookies Sora.com mints that are not the session cookie. We can ignore these. +/// This list is very likely to change over time. +const IRRELEVANT_COOKIE_NAMES: Lazy> = Lazy::new(|| { + HashSet::from([ + "cf_clearance", + "oai-did", + "_cfuvid", + "__cflb", + "__cf_bm", + "__Host-authjs.csrf-token", + "__Secure-authjs.callback-url", + "__Secure-authjs.pkce.code_verifier", + "__Secure-authjs.state", + ]) +}); + +/// Super naive heuristic for "logged out" total cookie length being ~1972 characters. +const SESSION_COOKIE_LENGTH_HEURISTIC : usize = 2000; + +//pub fn has_session_cookie(cookies: &str) -> bool { +// // NB: This is not a valid check for cookies, just a cheap heuristic. +// cookies.contains(SESSION_COOKIE_NAME) +//} + +#[derive(Copy,Clone,Eq,PartialEq,Debug)] +pub enum SessionCookiePresence { + Present, + MaybePresent, + Absent, +} + +/// OpenAI might change the name of the session cookie, and we don't want to break clients, +/// so we use some heuristics to determine if the cookie might be present. +pub fn has_session_cookie(cookies_header: &str) -> AnyhowResult { + let cookies_header = cookies_header.trim(); + if (cookies_header.is_empty()) { + return Ok(SessionCookiePresence::Absent); + } + + let cookies = Cookie::split_parse(cookies_header); + let mut unknown_cookie_count = 0; // NB: In the event they change their session cookie + + for cookie in cookies { + let cookie= cookie?; + let cookie_name = cookie.name(); + if cookie_name == SESSION_COOKIE_NAME { + return Ok(SessionCookiePresence::Present); + } else if !IRRELEVANT_COOKIE_NAMES.contains(cookie_name) { + unknown_cookie_count += 1; + } + } + + // This is a really stupid heuristic, but we want to be safe. + if cookies_header.len() > SESSION_COOKIE_LENGTH_HEURISTIC { + return Ok(SessionCookiePresence::MaybePresent); + } + + // Another stupid heuristic. + if unknown_cookie_count > 3 { + return Ok(SessionCookiePresence::MaybePresent); + } + + Ok(SessionCookiePresence::Absent) +} + +#[cfg(test)] +mod tests { + use crate::utils::has_session_cookie::{has_session_cookie, SessionCookiePresence}; + + #[test] + fn test() { + assert_eq!(has_session_cookie("").unwrap(), SessionCookiePresence::Absent); + assert_eq!(has_session_cookie("foo=bar").unwrap(), SessionCookiePresence::Absent); + + // Heuristic: unknown cookies. + assert_eq!(has_session_cookie("foo=bar;bin=baz;bat=ban;bash=barn").unwrap(), SessionCookiePresence::MaybePresent); + + // Heuristic: cookie length + let cookies = format!("cookie={}", "a".repeat(10000)); + assert_eq!(has_session_cookie(&cookies).unwrap(), SessionCookiePresence::MaybePresent); + + let cookies = "__Host-authjs.csrf-token=a2783b5f89ee8dcb184f64723c3fdc91868f89840cb842fb20c3bd3\ + d6b087d97%7C8e85a84ac3870dfab160d96c45fb8a0430350ae4b4755784c03c3b97a927eb06; __Secure-authjs.c\ + allback-url=https%3A%2F%2Fsora.com%2F; __cflb=0H28vtyuPgbsDEUozcoHSWoAnvpSqAh9PUFn8MimKwT; __Se\ + cure-authjs.session-token=eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2Q0JDLUhTNTEyIiwia2lkIjoiYWNBYVdTWHBfa\ + DZnNkxtUEs1S0RvTjR3NHFxTEsxVlpSTHA4VWdiaFdWNjctMG5pbDdVVjBCN2dSUU9Bem84ZXBjMVFjb1ozNGxwa0c1OEUy\ + U2Q4cUEifQ..I4nYqp8dqgnuBwIgzbtoTg.etnsxNRheMCGgAFaTzAxGdqjao68Xf_NCHqv8h5ZBhfDx_b0dg2bzDIfNFF5\ + _8VG1dwZkqe83k3JrOejBVKi3wDSI5p_EgU8YKf_oxshusBn09mGZmAPd49_F73DARypF0eHKyUnGfelkFmVW610P0nV3Pn\ + KNJhjIHHxAh_9dHnRvQ5WtaYMvSTE5Yc8Ut1VwKs9QmYdTkfcC_gDATQ-dgF8eew5Seh4KALizGcQqYhcENMVzfshSMwNrr\ + mZ-RL6X71wTqhKV3XiJnp_s7e6DxRfcb25V-aOD_rsVa1uAxpYzYvc197Cd0psqQ1MpPjGohUSXjwg177FvBmloAxy8-Qbf\ + Moz9ofugywgEqQBJIEznddwEOScVqoOhRbYXGpqBY-rU5ChDG4qetbfKNCX2DT0R0lShR1n-UXo-XJJuuFJ-FRPTOGMVJHX\ + 5CT-k258ybtsMBM7sMQhohpjMnJuSZkloj0L6gZltG21syJy-14X4ykU20lfneVn-yKorTo2JbDMWUvWhdEIVt5-5czXmA7\ + csWiqmCgrkSHsw0NescOtLT1VfqW2aAd6X8ShsDzyMjfBC5GW84S1uBUURbr0WkcxLZFb_IsKaAA0xsfUri0rHXi35rabhM\ + KgH4mn8RztNsyod-tEwLil1LZS2HmoyB8RAC1xh5HIhDe3RJYaHyCdXJqeZjb1KBC1xDQV2QbQZ91y0JZzBXvg6NTFYJ-wb\ + CytyOnEbknHG_AM0p-GvxjckkL1PbasHdiOOKyT49hYCEQF0Zrnmq4DlvO46CpLBvpENC3NlqDJ3d3RD9_-WVn1yjVHI4Yl\ + VPFAyrYnfhtRF7WlgBPOnq6mPDF17dSIdLLWH3iQQQFfq-zxULLn0RaHWgeWndOaOLVPLe6fxZ_6-_lFLR-Vi-DiJj8dAM4\ + lRTUucyoXv93vqzCxN653V-LOTmg7SFpe2OXFsmXHiufoFQcVm-erpNZ3_hlxOOdYJh8wuQUBUKPhwqunTcIHZIr2QqjmnX\ + SSmBPw1-HQtWkEVGA5qKWLwciaX74Z8NtUWnbC4ZmYGTbnirIbsZ0EyYRNDvNjNGMoWrJBCyi_K3uP6g1GT-Zhc123dGZqu\ + 1REqPiCmGK8l3ofOHEVFFG7k-Yc80RiVI7EQ2i5qN7nJEavDetXuC-nlbTbYqjqEut31T9G-IC3HRkHg7HLEBIOVYKbPLe5\ + BNzfAYO48YlBc1CLRFcx5LQMM1X_hS_WKLtYPkGxE7BZnR11Acf904kELPbL1-7-ZXnAYiQodwa9wzS6uLqdy6FCuqOG08E\ + K2mCDsIZXw3mjqOlDLXSSLNmu6CH5r3aXcnVaYHJQpSryPfNpNltXdYnrxF2CgWHPAOuleCKHY6nH9aKIH6RsFN7ZhgLZl4\ + 5NnVzU3dx5PiSuwELiXFbziCXqfcbbUrhfwnJKPMxQqxErqadM6u9b4hB4QTZq41_OEu3S3Y8y2E1aGPZ81IMK4QRb5A4wh\ + WR9gBGF1FUud9JB5kfI479rpcwlDtoCxTkaixTsU0DwNzqA0gZvXZbJAqtSjyJIhTU_OBz94YzMsYYbuLSi82dm_a90zV6n\ + EnCYjuPuS6F2nUgW7BhNRGzusP7oIiECCJM8US2iWAGcQffhsEiwmB7YxET3gTifxNv6TI96muD0qA0G7zoNJkTQ11jxnWk\ + PqDHH1Mk545Ge7hL3YvjjFcJwhypNHxJCicQeuqOuzJtiS8DAzC1T7xS6ZbpFi8vcXmmv439RKYw4FoDfkl3JA8dBLM-Dii\ + 2UlSX0byjppcjDOVX1G4azYoJawXhAchOFTkh7ZgAmAC-XIQYN3KZIwUpWIbnkTt-8V-SQlf56YW6VNNP7dAYM8rhqC7quC\ + QfFicZYH32oTenr6-75Bq6dFGCdF4Ieh4M1eVI8tgaOM9X-kVy5gliQ61gyyPFiVLGkPJQfJT6jWPVAu1JrB6v0mNm7G10m\ + qUPptxw38K_75Mvqd4D5ye6JUAii9iCMbZ8YMZHD4ncum-WJ2cDofssiXM_psgB6W94-PxxFZxfzL8MIqKe1ozAA8pJIyVu\ + gqm-AYcFJprbW3C6cgTfeZNW1_4k0stDgE3N2jCVLvGMKkiGTYC_SwgzqF3DU8Up0p_jMoJQ9hh3re7fjEGuKvtIfc_jeFq\ + c9-q1LsVirGizzhS_VTA5Ltnh4quwaIU3zkI_y3N9xV62ijUectvOPWONl42Qm9NUU4fQZynrUbTd9sSjYRNHfveIMDhbCj\ + dDWjzHjMgCeUopSyrLLra1uC0NsJrBoq24QMQ00jHgKWOnqpdFTjPopZ-zDcN0E4FkYB8kKbrz4u4ZIn0fNNnE2bzlU8rbH"; + + assert_eq!(has_session_cookie(cookies).unwrap(), SessionCookiePresence::Present); + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/utils/lightweight_sora_jwt_parse.rs b/crates/lib/api_clients/openai_sora_client/src/utils/lightweight_sora_jwt_parse.rs new file mode 100644 index 0000000000..a1605e82cb --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/utils/lightweight_sora_jwt_parse.rs @@ -0,0 +1,139 @@ +//! Adapted from 'jwt' crate + +use anyhow::anyhow; +use base64::Engine; +use base64::prelude::BASE64_URL_SAFE_NO_PAD; +use chrono::{DateTime, Utc}; +use log::warn; +use serde_json::Value; +use errors::AnyhowResult; + +const SEPARATOR: &str = "."; + +#[derive(Clone, Debug)] +pub struct SoraJwtClaims { + pub created: DateTime, + pub expiration: DateTime, + pub user_id: String, + pub email: String, + pub email_verified: bool, +} + +pub fn lightweight_sora_jwt_parse(token: &str) -> AnyhowResult { + let [_header_str, claims_str, _signature_str] = split_components(token)?; + + let claims = decode_base64(claims_str)?; + let claims : serde_json::Map = serde_json::from_str(&claims)?; + + let iat = claims.get("iat") + .map(|val| val.as_number()) + .flatten() + .ok_or(anyhow!("no iat claim"))?; + + let iat = iat.as_i64() + .ok_or(anyhow!("iat is not a number"))?; + + let iat = DateTime::from_timestamp(iat, 0) + .ok_or(anyhow!("iat is not a valid timestamp"))?; + + let exp = claims.get("exp") + .map(|val| val.as_number()) + .flatten() + .ok_or(anyhow!("no exp claim"))?; + + let exp = exp.as_i64() + .ok_or(anyhow!("exp is not a number"))?; + + let exp = DateTime::from_timestamp(exp, 0) + .ok_or(anyhow!("iat is not a valid timestamp"))?; + + let user_id = claims.get("https://api.openai.com/auth") + .map(|val| val.get("user_id")) + .flatten() + .map(|val| val.as_str()) + .flatten() + .ok_or(anyhow!("no user_id claim"))?; + + let profile = claims.get("https://api.openai.com/profile") + .ok_or(anyhow!("no user_id claim"))?; + + let email = profile.get("email") + .map(|val| val.as_str()) + .flatten() + .ok_or(anyhow!("no email claim"))?; + + let email_verified = profile.get("email_verified") + .map(|val| val.as_bool()) + .flatten() + .unwrap_or_else(|| { + warn!("no email_verified claim"); + false + }); + + Ok(SoraJwtClaims { + created: iat, + expiration: exp, + user_id: user_id.to_string(), + email: email.to_string(), + email_verified, + }) +} + +fn split_components(token: &str) -> AnyhowResult<[&str; 3]> { + let mut components = token.split(SEPARATOR); + let header = components.next().ok_or(anyhow!("no header component"))?; + let claims = components.next().ok_or(anyhow!("no claims component"))?; + let signature = components.next().ok_or(anyhow!("no signature component"))?; + + if components.next().is_some() { + return Err(anyhow!("too many components")); + } + + Ok([header, claims, signature]) +} + +fn decode_base64(raw: &str) -> AnyhowResult { + let json_bytes = BASE64_URL_SAFE_NO_PAD.decode(raw)?; + let json_str = String::from_utf8(json_bytes.clone())?; + Ok(json_str) +} + +#[cfg(test)] +mod tests { + use crate::utils::lightweight_sora_jwt_parse::lightweight_sora_jwt_parse; + + #[test] + fn test_sora_jwt_bearer_token() { + let token = + "eyJhbGciOiJSUzI1NiIsImtpZCI6IjE5MzQ0ZTY1LWJiYzktNDRkMS1hOWQwLWY5NTdiMDc5YmQwZSIsInR5cCI6Ik\ + pXVCJ9.eyJhdWQiOlsiaHR0cHM6Ly9hcGkub3BlbmFpLmNvbS92MSJdLCJjbGllbnRfaWQiOiJhcHBfTTFuUTN0UjV2\ + VzdYOWpMMnBFNmdIOGRLNCIsImV4cCI6MTc0NTgwMDQ3MiwiaHR0cHM6Ly9hcGkub3BlbmFpLmNvbS9hdXRoIjp7InV\ + zZXJfaWQiOiJ1c2VyLTZOeEpmbEFIb0VCREp6Wmw5aVhocWJERyJ9LCJodHRwczovL2FwaS5vcGVuYWkuY29tL3Byb2\ + ZpbGUiOnsiZW1haWwiOiJ2b2NvZGVzMjAyMEBnbWFpbC5jb20iLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZX0sImlhdCI6M\ + Tc0NDkzNjQ3MSwiaXNzIjoiaHR0cHM6Ly9hdXRoLm9wZW5haS5jb20iLCJqdGkiOiI0NTZmZjY5Yi0xZTZhLTQzOWYt\ + YTYzMC1jNmFjYzI3NzU1YjAiLCJuYmYiOjE3NDQ5MzY0NzEsInB3ZF9hdXRoX3RpbWUiOjE3NDQwNTkzMTc3NTAsInN\ + jcCI6WyJvcGVuaWQiLCJlbWFpbCIsInByb2ZpbGUiLCJvZmZsaW5lX2FjY2VzcyJdLCJzZXNzaW9uX2lkIjoiYXV0aH\ + Nlc3NfcEE4UGl3V0trNmJsSE1qQnVCQnVSRmh2Iiwic3ViIjoiZ29vZ2xlLW9hdXRoMnwxMTMxMDE5Njc2MTIzOTY3O\ + TM3NzcifQ.JOxYfrQSIsHdf-VN1zcOqDOS3tF1HpUYnRVfPKgi65am3sNiniOM1kWTNrKE5-sh4DaZ33G_XniGkhg-9\ + QKF8tspXpZ6BnTNkf-QdGqR_v89AIO8pZgQ7oECLXhnmg-31G-LAHlGfwWedJ6qPVy4dL5zoR0wVyoQ8gYv5QzQC-zd\ + IF7uY-umKKloAAYP_tkKLlQHackjTynN-bpu2mKv55-h7a3hSEPfxvgX0SsvL69xtNsZkwb43bqyP1x_c2ErfYoPyzg\ + rqgMIRZjCw9TI33vISKM_BfQXBiZT_BHkX4s0Jng3ph4vrwvnBsa0HpNxBhQYKb1u9gZXs7XjLZhjZJZsDTQI1V4Wov\ + iAL7ihvZkLxJRtDCOmX7-5BTEsMInFgXUAhCDhQUMwSfLLJuinBS96NGSX5_TfIVMkbCl-HWzdQx7KZAsahNe_CmuYT\ + lZH32NuRn78ohY4eViRyfC8OXVLwoLxxpen2CWlV2dxV5wwXeij_kHt5wOXSGYpSfxz37cK0CaK8K2pwudasxQ0lv2V\ + _Xx_z4UaEOr5wqT_BK1A1HO4HRjvFX2soF-qm4GCSWAbmFZs7DVEClrotxRx2romifRVb8XRia2M6YTmDyIT0YphFgG\ + nnrDIZNKO23uDYT60LVMVYpCCY2RiwxCR9bn7AeGbmi7cpL9aYZ5uJRM"; + + // jwt payload : + // - "https://api.openai.com/auth".user_id = "user-6NxJflAHoEBDJzZl9iXhqbDG" + // - "https://api.openai.com/profile".email = "vocodes2020@gmail.com" + // - "https://api.openai.com/profile".email_verified = true + // - "iat": 1744936471, - issued at claim (5 days ago) + // - "nbf": 1744936471, - not before, don't use before date + // - "exp": 1745800472, - in 5 days, expiry + + let result = lightweight_sora_jwt_parse(token).expect("claim should parse"); + + assert_eq!(result.email, "vocodes2020@gmail.com"); + assert_eq!(result.email_verified, true); + } +} diff --git a/crates/lib/api_clients/openai_sora_client/src/utils/mod.rs b/crates/lib/api_clients/openai_sora_client/src/utils/mod.rs new file mode 100644 index 0000000000..f46402e68c --- /dev/null +++ b/crates/lib/api_clients/openai_sora_client/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod has_session_cookie; +pub mod lightweight_sora_jwt_parse; \ No newline at end of file diff --git a/crates/lib/elasticsearch_schema/src/searches/search_model_weights/search_model_weights.rs b/crates/lib/elasticsearch_schema/src/searches/search_model_weights/search_model_weights.rs index 6d2129a843..d7ec87aab4 100644 --- a/crates/lib/elasticsearch_schema/src/searches/search_model_weights/search_model_weights.rs +++ b/crates/lib/elasticsearch_schema/src/searches/search_model_weights/search_model_weights.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use elasticsearch::{Elasticsearch, SearchParts}; +use log::debug; use once_cell::sync::Lazy; use serde_json::{json, Number, Value}; @@ -75,7 +76,7 @@ pub async fn search_model_weights(args: SearchArgs<'_>) -> AnyhowResult", @@ -18,15 +18,16 @@ path = "src/lib.rs" normal = ["workspace-build-acceleration"] [dependencies] +# Workspace hack (faster compile times) +workspace-build-acceleration = { workspace = true } # Internal -errors = { path = "../../errors" } - -# Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } +errors = { workspace = true} +mimetypes = { workspace = true } # External -image = "0.25.1" +image = "0.25.6" [dev-dependencies] +anyhow = { workspace = true } testing = { path = "../../testing" } diff --git a/crates/lib/files/images/src/image_info/image_info.rs b/crates/lib/files/images/src/image_info/image_info.rs new file mode 100644 index 0000000000..600101c349 --- /dev/null +++ b/crates/lib/files/images/src/image_info/image_info.rs @@ -0,0 +1,100 @@ +use crate::image_info::image_info_error::ImageInfoError; +use image::{GenericImageView, ImageReader}; +use mimetypes::mimetype_info::file_extension::FileExtension; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use std::io::Cursor; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct ImageInfo { + /// The mimetype we determined + mimetype_info: MimetypeInfo, + + /// The width of the image in pixels + width: u32, + + /// The height of the image in pixels + height: u32, +} + +impl ImageInfo { + /// Note: This has to do a full file read, and we throw the parsed image on the floor. + pub fn decode_image_from_bytes(bytes: &[u8]) -> Result { + let image = ImageReader::new(Cursor::new(bytes)).with_guessed_format()?.decode()?; + let (width, height) = image.dimensions(); + Ok(ImageInfo { + mimetype_info: MimetypeInfo::get_for_bytes(bytes) + .ok_or_else(|| ImageInfoError::CouldNotDetermineMimetype)?, + width, + height, + }) + } + + /// Note: This has to do a full file read, and we throw the parsed image on the floor. + pub fn decode_image_from_path>(path: P) -> Result { + let image = ImageReader::open(&path)?.decode()?; + let (width, height) = image.dimensions(); + Ok(ImageInfo { + mimetype_info: MimetypeInfo::get_for_path(path)? + .ok_or_else(|| ImageInfoError::CouldNotDetermineMimetype)?, + width, + height, + }) + } + + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } + + pub fn dimensions(&self) -> (u32, u32) { + (self.width, self.height) + } + + pub fn mime_type(&self) -> &str { + self.mimetype_info.mime_type() + } + + pub fn file_extension(&self) -> Option { + self.mimetype_info.file_extension() + } +} + +#[cfg(test)] +mod tests { + use crate::image_info::image_info::ImageInfo; + use mimetypes::mimetype_info::file_extension::FileExtension; + use testing::test_file_path::test_file_path; + + #[test] + fn jpg_file() -> anyhow::Result<()> { + let path = test_file_path("test_data/image/mochi.jpg")?; + let info = ImageInfo::decode_image_from_path(path)?; + assert_eq!(info.width(), 753); + assert_eq!(info.height(), 1000); + assert_eq!(info.mime_type(), "image/jpeg"); + let ext = info.file_extension().expect("should have extension"); + assert_eq!(ext, FileExtension::Jpg); + assert_eq!(ext.extension_with_period(), ".jpg"); + assert_eq!(ext.extension_without_period(), "jpg"); + Ok(()) + } + + #[test] + fn jpg_bytes() -> anyhow::Result<()> { + let path = test_file_path("test_data/image/juno.jpg")?; + let bytes = std::fs::read(path)?; + let info = ImageInfo::decode_image_from_bytes(&bytes)?; + assert_eq!(info.width(), 1000); + assert_eq!(info.height(), 750); + assert_eq!(info.mime_type(), "image/jpeg"); + let ext = info.file_extension().expect("should have extension"); + assert_eq!(ext, FileExtension::Jpg); + assert_eq!(ext.extension_with_period(), ".jpg"); + assert_eq!(ext.extension_without_period(), "jpg"); + Ok(()) + } +} diff --git a/crates/lib/files/images/src/image_info/image_info_error.rs b/crates/lib/files/images/src/image_info/image_info_error.rs new file mode 100644 index 0000000000..e1d84cd3c9 --- /dev/null +++ b/crates/lib/files/images/src/image_info/image_info_error.rs @@ -0,0 +1,39 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::io; + +#[derive(Debug)] +pub enum ImageInfoError { + CouldNotDetermineMimetype, + ImageError(image::ImageError), + IoError(io::Error), +} + +impl Error for ImageInfoError {} + +impl Display for ImageInfoError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ImageInfoError::CouldNotDetermineMimetype => { + write!(f, "ImageInfoError::CouldNotDetermineMimetype") + } + ImageInfoError::ImageError(err) => { + write!(f, "ImageInfoError::ImageError: {}", err) + } + ImageInfoError::IoError(err) => { + write!(f, "ImageInfoError::IoError: {}", err) + } + } + } +} + +impl From for ImageInfoError { + fn from(err: image::ImageError) -> Self { + ImageInfoError::ImageError(err) + } +} +impl From for ImageInfoError { + fn from(err: io::Error) -> Self { + ImageInfoError::IoError(err) + } +} diff --git a/crates/lib/files/images/src/image_info/mod.rs b/crates/lib/files/images/src/image_info/mod.rs new file mode 100644 index 0000000000..1d5cdbac30 --- /dev/null +++ b/crates/lib/files/images/src/image_info/mod.rs @@ -0,0 +1,2 @@ +pub mod image_info; +pub mod image_info_error; \ No newline at end of file diff --git a/crates/lib/files/images/src/lib.rs b/crates/lib/files/images/src/lib.rs index 1d32966677..40dca2f3b8 100644 --- a/crates/lib/files/images/src/lib.rs +++ b/crates/lib/files/images/src/lib.rs @@ -23,6 +23,7 @@ // Re-export image to everyone importing this library pub use image; +pub mod image_info; pub mod resize_image_file_preserving_aspect; pub mod resize_preserving_aspect; diff --git a/crates/lib/files/media/Cargo.toml b/crates/lib/files/media/Cargo.toml index 58b7233bf8..a3ac2eca09 100644 --- a/crates/lib/files/media/Cargo.toml +++ b/crates/lib/files/media/Cargo.toml @@ -16,15 +16,14 @@ path = "src/lib.rs" normal = ["workspace-build-acceleration"] [dependencies] +# Workspace hack (faster compile times) +workspace-build-acceleration = { workspace = true } # Internal -errors = { path = "../../errors" } - -# Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } +errors = { workspace = true } # External -log = "0.4.14" +log = { workspace = true } # Powerful audio decoder and demuxer #symphonia = { version = "0.5.2", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "vorbis", "wav"] } diff --git a/crates/lib/files/mimetypes/Cargo.toml b/crates/lib/files/mimetypes/Cargo.toml index 9555ce82f1..83105b766f 100644 --- a/crates/lib/files/mimetypes/Cargo.toml +++ b/crates/lib/files/mimetypes/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mimetypes" -edition = "2018" +edition = "2024" version = "0.0.1" authors = [ "Brandon Thomas ", @@ -18,11 +18,11 @@ normal = ["workspace-build-acceleration"] [dependencies] # Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } +workspace-build-acceleration = { workspace = true } # External -infer = "0.16.0" # Mimetype detection - +infer = "0.19.0" # Mimetype detection [dev-dependencies] -# None yet +testing = { path = "../../testing" } +anyhow = { workspace = true } diff --git a/crates/lib/files/mimetypes/src/lib.rs b/crates/lib/files/mimetypes/src/lib.rs index 194f6fe739..4d04fd6188 100644 --- a/crates/lib/files/mimetypes/src/lib.rs +++ b/crates/lib/files/mimetypes/src/lib.rs @@ -21,4 +21,5 @@ pub mod mimetype_for_bytes; pub mod mimetype_for_file; +pub mod mimetype_info; pub mod mimetype_to_extension; diff --git a/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs b/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs new file mode 100644 index 0000000000..4b88194ef5 --- /dev/null +++ b/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs @@ -0,0 +1,68 @@ + +/// This is a constrained set of file extensions that: +/// 1. we care about +/// 2. we can detect with magic bytes +/// 3. are not text formats / are binary formats +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub enum FileExtension { + Aac, + Flac, + Gif, + Jpg, + Mov, + Mp3, + Mp4, + Ogg, + Opus, + Png, + Wav, + Webm, + Webp, +} + +impl FileExtension { + pub fn from_mimetype(mimetype: &str) -> Option { + match mimetype { + "audio/aac" => Some(FileExtension::Aac), + "audio/flac" => Some(FileExtension::Flac), + "image/gif" => Some(FileExtension::Gif), + "image/jpeg" => Some(FileExtension::Jpg), + "video/quicktime" => Some(FileExtension::Mov), + "audio/mpeg" => Some(FileExtension::Mp3), + "video/mp4" => Some(FileExtension::Mp4), + "audio/ogg" => Some(FileExtension::Ogg), + "audio/opus" => Some(FileExtension::Opus), + "image/png" => Some(FileExtension::Png), + "audio/wav" => Some(FileExtension::Wav), + "video/webm" => Some(FileExtension::Webm), + "image/webp" => Some(FileExtension::Webp), + _ => None, + } + } + + pub fn extension_with_period(&self) -> &'static str { + self.with_and_without_period().0 + } + + pub fn extension_without_period(&self) -> &'static str { + self.with_and_without_period().1 + } + + fn with_and_without_period(&self) -> (&'static str, &'static str) { + match self { + FileExtension::Aac => (".aac", "aac"), + FileExtension::Flac => (".flac", "flac"), + FileExtension::Gif => (".gif", "gif"), + FileExtension::Jpg => (".jpg", "jpg"), + FileExtension::Mov => (".mov", "mov"), + FileExtension::Mp3 => (".mp3", "mp3"), + FileExtension::Mp4 => (".mp4", "mp4"), + FileExtension::Ogg => (".ogg", "ogg"), + FileExtension::Opus => (".opus", "opus"), + FileExtension::Png => (".png", "png"), + FileExtension::Wav => (".wav", "wav"), + FileExtension::Webm => (".webm", "webm"), + FileExtension::Webp => (".webp", "webp"), + } + } +} diff --git a/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs b/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs new file mode 100644 index 0000000000..c8336bc908 --- /dev/null +++ b/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs @@ -0,0 +1,80 @@ +use crate::mimetype_info::file_extension::FileExtension; +use infer::Type; +use std::io; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct MimetypeInfo { + /// The mimetype we determined + mime_type: String, + + /// The extension for the mimetype, if we could determine it. + file_extension: Option, +} + +impl MimetypeInfo { + pub fn get_for_bytes(bytes: &[u8]) -> Option { + infer::get(bytes) + .map(|t| Self::for_type(t)) + } + + pub fn get_for_path>(path: P) -> io::Result> { + infer::get_from_path(path) + .map(|maybe_type| maybe_type.map(|t| Self::for_type(t))) + } + + fn for_type(r#type: Type) -> Self { + let mime_type = r#type.mime_type().to_string(); + MimetypeInfo { + file_extension: FileExtension::from_mimetype(&mime_type), + mime_type, + } + } + + pub fn mime_type(&self) -> &str { + &self.mime_type + } + + pub fn file_extension(&self) -> Option { + self.file_extension + } +} + +#[cfg(test)] +mod tests { + use crate::mimetype_info::file_extension::FileExtension; + use crate::mimetype_info::mimetype_info::MimetypeInfo; + + mod for_bytes { + use super::*; + + #[test] + fn png() { + // NB: From "Hanashi Mask.png" + let bytes: Vec = vec![137, 80, 78, 71, 13, 10, 26, 10]; + let t = MimetypeInfo::get_for_bytes(&bytes).expect("should detect bytes"); + assert_eq!(t.mime_type(), "image/png"); + let ext = t.file_extension.expect("should have extension"); + assert_eq!(ext, FileExtension::Png); + assert_eq!(ext.extension_with_period(), ".png"); + assert_eq!(ext.extension_without_period(), "png"); + } + } + + mod for_files { + use testing::test_file_path::test_file_path; + use super::*; + + #[test] + fn mp3() -> anyhow::Result<()> { + let path = test_file_path("test_data/audio/mp3/super_mario_rpg_beware_the_forests_mushrooms.mp3")?; + let t = MimetypeInfo::get_for_path(path)?.expect("should detect type"); + assert_eq!(t.mime_type(), "audio/mpeg"); + let ext = t.file_extension.expect("should have extension"); + assert_eq!(ext, FileExtension::Mp3); + assert_eq!(ext.extension_with_period(), ".mp3"); + assert_eq!(ext.extension_without_period(), "mp3"); + Ok(()) + } + } +} diff --git a/crates/lib/files/mimetypes/src/mimetype_info/mod.rs b/crates/lib/files/mimetypes/src/mimetype_info/mod.rs new file mode 100644 index 0000000000..910eaea54f --- /dev/null +++ b/crates/lib/files/mimetypes/src/mimetype_info/mod.rs @@ -0,0 +1,2 @@ +pub mod file_extension; +pub mod mimetype_info; \ No newline at end of file diff --git a/crates/lib/files/videos/Cargo.toml b/crates/lib/files/videos/Cargo.toml index 7434815509..68ba3fb656 100644 --- a/crates/lib/files/videos/Cargo.toml +++ b/crates/lib/files/videos/Cargo.toml @@ -18,14 +18,14 @@ normal = ["workspace-build-acceleration"] [dependencies] # Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } +workspace-build-acceleration = { workspace = true } # Internal -errors = { path = "../../errors" } +errors = { workspace = true } # External ffprobe = "0.4.0" -log = "0.4.20" +log = { workspace = true } # TODO(bt,2024-08-08): We might drop `mp4` crate because it crashes on reading # some videos with the message "avcc not found" mp4 = "0.14.0" diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/mark_generic_inference_job_successfully_done_by_token.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/mark_generic_inference_job_successfully_done_by_token.rs new file mode 100644 index 0000000000..3b192321f3 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/mark_generic_inference_job_successfully_done_by_token.rs @@ -0,0 +1,57 @@ +use std::time::Duration; + +use anyhow::anyhow; +use sqlx::MySqlPool; + +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use errors::AnyhowResult; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; + +pub async fn mark_generic_inference_job_successfully_done_by_token( + pool: &MySqlPool, + token: &InferenceJobToken, + maybe_entity_type: Option, + maybe_entity_token: Option<&str>, + total_job_duration: Option, + inference_duration: Option, +) -> AnyhowResult<()> +{ + // NB: MySql's unsigned int (32 bits) can store integers up to 4,294,967,295. + // Given milliseconds, this is ~49.71 days, which should be plenty for us. + let truncated_total_job_execution_millis = total_job_duration + .map(|duration| duration.as_millis() as u32) + .unwrap_or(0); + let truncated_inference_execution_millis = inference_duration + .map(|duration| duration.as_millis() as u32) + .unwrap_or(0); + + let query_result = sqlx::query!( + r#" +UPDATE generic_inference_jobs +SET + status = "complete_success", + on_success_result_entity_type = ?, + on_success_result_entity_token = ?, + failure_reason = NULL, + internal_debugging_failure_reason = NULL, + success_execution_millis = ?, + success_inference_execution_millis = ?, + retry_at = NULL, + successfully_completed_at = NOW() +WHERE token = ? + "#, + maybe_entity_type, + maybe_entity_token, + truncated_total_job_execution_millis, + truncated_inference_execution_millis, + token.as_str() + ) + .execute(pool) + .await; + + match query_result { + Err(err) => Err(anyhow!("error with query: {:?}", err)), + Ok(_r) => Ok(()), + } +} diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs index 37b2b2d2bf..e952070de3 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs @@ -7,4 +7,5 @@ pub mod job_status; pub mod kill_generic_inference_jobs; pub mod kill_jobs_in_development; pub mod list_session_jobs; -pub mod mark_generic_inference_job_cancelled_by_user; \ No newline at end of file +pub mod mark_generic_inference_job_cancelled_by_user; +pub mod mark_generic_inference_job_successfully_done_by_token; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs new file mode 100644 index 0000000000..8d02c6ec49 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs @@ -0,0 +1,269 @@ +use crate::queries::media_files::create::generic_insert::insert_media_file_generic::{insert_media_file_generic, InsertArgs}; +use anyhow::anyhow; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use sqlx::MySqlPool; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; +use crate::queries::media_files::create::insert_builder::media_file_insert_builder_error::MediaFileInsertBuilderError; + +/// Builder for inserting a media files into the database. +/// Resembles an ORM, but can still use statically type checked queries (which is probably +/// really important since we don't have tests on database functionality yet). +#[derive(Clone)] +pub struct MediaFileInsertBuilder { + // TODO: Allow specifying the token externally + // token: MediaFileToken, + + // Creator info + maybe_creator_user_token: Option, + maybe_creator_anonymous_visitor_token: Option, + creator_ip_address: Option, // NB: Non-nullable field + creator_set_visibility: Visibility, // NB: Non-nullable field + + // Important database indices + media_file_class: MediaFileClass, // NB: Non-nullable field + media_file_type: Option, // NB: Non-nullable field + is_user_upload: bool, + is_intermediate_system_file: bool, + + // Product and other origination information + origin_category: Option, // NB: Non-nullable field + origin_product_category: MediaFileOriginProductCategory, // NB: Non-nullable field + // maybe_origin_model_type: Option, + // maybe_origin_model_token: Option<&'a ModelWeightToken>, + // maybe_origin_filename: Option, + + // Media info + maybe_mime_type: Option, + file_size_bytes: u64, // NB: Non-nullable field + // maybe_duration_millis: Option, + // maybe_audio_encoding: Option<&'a str>, + // maybe_video_encoding: Option<&'a str>, + maybe_frame_width: Option, + maybe_frame_height: Option, + checksum_sha2: Option, // NB: Non-nullable field + + // // User text information + // maybe_title: Option<&'a str>, + // maybe_text_transcript: Option<&'a str>, + + // // If generated from a scene, this is the scene media file token. + // maybe_scene_source_media_file_token: Option<&'a MediaFileToken>, + + // // If additional prompt details are stored, this is the prompt token. + // maybe_prompt_token: Option<&'a PromptToken>, + + // // If batch generated, this is the batch token. + // maybe_batch_token: Option<&'a BatchGenerationToken>, + + // Storage details + public_bucket_directory_hash: Option, // NB: Non-nullable field(s) + + // Remaining fields... +} + +impl MediaFileInsertBuilder { + //pub fn generate_token() -> Self { + // Self::with_token(&MediaFileToken::generate()) + //} + + pub fn new() -> Self { + MediaFileInsertBuilder { + //token: token.clone(), + maybe_creator_user_token: None, + maybe_creator_anonymous_visitor_token: None, + creator_ip_address: None, + creator_set_visibility: Visibility::Public, + media_file_class: MediaFileClass::Unknown, + media_file_type: None, + is_user_upload: false, + is_intermediate_system_file: false, + origin_category: None, + origin_product_category: MediaFileOriginProductCategory::Unknown, + maybe_mime_type: None, + file_size_bytes: 0, + maybe_frame_width: None, + maybe_frame_height: None, + checksum_sha2: None, + public_bucket_directory_hash: None, + } + } + + pub fn creator_user(mut self, user_token: &UserToken) -> Self { + self.maybe_creator_user_token = Some(user_token.clone()); + self + } + + pub fn maybe_creator_user(mut self, maybe_user: Option<&UserToken>) -> Self { + self.maybe_creator_user_token = maybe_user.map(|token| token.clone()); + self + } + + pub fn creator_anonymous_visitor(mut self, avt: &AnonymousVisitorTrackingToken) -> Self { + self.maybe_creator_anonymous_visitor_token = Some(avt.clone()); + self + } + + pub fn maybe_creator_anonymous_visitor(mut self, maybe_avt: Option<&AnonymousVisitorTrackingToken>) -> Self { + self.maybe_creator_anonymous_visitor_token = maybe_avt.map(|token| token.clone()); + self + } + + pub fn creator_ip_address(mut self, ip_address: &str) -> Self { + self.creator_ip_address = Some(ip_address.to_string()); + self + } + + pub fn creator_set_visibility(mut self, visibility: Visibility) -> Self { + self.creator_set_visibility = visibility; + self + } + + pub fn media_file_class(mut self, media_file_class: MediaFileClass) -> Self { + self.media_file_class = media_file_class; + self + } + + pub fn media_file_type(mut self, media_file_type: MediaFileType) -> Self { + self.media_file_type = Some(media_file_type); + self + } + + #[allow(clippy::wrong_self_convention)] + pub fn is_user_upload(mut self, is_user_upload: bool) -> Self { + self.is_user_upload = is_user_upload; + self + } + + #[allow(clippy::wrong_self_convention)] + pub fn is_intermediate_system_file(mut self, is_intermediate_system_file: bool) -> Self { + self.is_intermediate_system_file = is_intermediate_system_file; + self + } + + pub fn media_file_origin_category(mut self, origin_category: MediaFileOriginCategory) -> Self { + self.origin_category = Some(origin_category); + self + } + + pub fn media_file_origin_product_category(mut self, origin_category: MediaFileOriginProductCategory) -> Self { + self.origin_product_category = origin_category; + self + } + + // TODO: Method for maybe_origin_model_type + // TODO: Method for maybe_origin_model_token + // TODO: Method for maybe_origin_filename + + pub fn mime_type(mut self, mime_type: &str) -> Self { + self.maybe_mime_type = Some(mime_type.to_string()); + self + } + + pub fn file_size_bytes(mut self, file_size_bytes: u64) -> Self { + self.file_size_bytes = file_size_bytes; + self + } + + // TODO: maybe_duration_millis + // TODO: maybe_audio_encoding + // TODO: maybe_video_encoding + + pub fn frame_width(mut self, frame_width: u32) -> Self { + self.maybe_frame_width = Some(frame_width); + self + } + + pub fn frame_height(mut self, frame_height: u32) -> Self { + self.maybe_frame_height = Some(frame_height); + self + } + + pub fn checksum_sha2(mut self, checksum_sha2: &str) -> Self { + self.checksum_sha2 = Some(checksum_sha2.to_string()); + self + } + + // TODO: maybe_title + // TODO: maybe_text_transcript + // TODO: maybe_scene_source_media_file_token + // TODO: maybe_prompt_token + // TODO: maybe_batch_token + + pub fn public_bucket_directory_hash(mut self, public_bucket_directory_hash: &MediaFileBucketPath) -> Self { + self.public_bucket_directory_hash = Some(public_bucket_directory_hash.clone()); + self + } + + // TODO(bt,2025-04-26): Other connector options. + pub async fn insert_pool(self, mysql_pool: &MySqlPool) -> Result { + let media_file_type = self.media_file_type + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Media file type is required".to_string()))?; + + let origin_category = self.origin_category + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Origin category is required".to_string()))?; + + let checksum_sha2 = self.checksum_sha2 + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Checksum SHA2 is required".to_string()))?; + + let bucket_path = self.public_bucket_directory_hash + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Public bucket directory hash is required".to_string()))?; + + let result = insert_media_file_generic(InsertArgs { + pool: mysql_pool, + maybe_creator_user_token: self.maybe_creator_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: self.maybe_creator_anonymous_visitor_token.as_ref(), + creator_ip_address: self.creator_ip_address.as_deref().unwrap_or("127.0.0.1"), + creator_set_visibility: self.creator_set_visibility, + media_class: self.media_file_class, + media_type: media_file_type, + is_user_upload: self.is_user_upload, + is_intermediate_system_file: self.is_intermediate_system_file, + origin_category, + origin_product_category: self.origin_product_category, + maybe_origin_model_type: None, // TODO + maybe_origin_model_token: None, // TODO + maybe_origin_filename: None, // TODO + maybe_mime_type: self.maybe_mime_type.as_deref(), + file_size_bytes: self.file_size_bytes, + maybe_duration_millis: None, // TODO + maybe_audio_encoding: None, // TODO + maybe_video_encoding: None, // TODO + maybe_frame_width: self.maybe_frame_width, + maybe_frame_height: self.maybe_frame_height, + checksum_sha2: &checksum_sha2, + maybe_title: None, // TODO + maybe_text_transcript: None, // TODO + maybe_scene_source_media_file_token: None, // TODO + maybe_prompt_token: None, // TODO + maybe_batch_token: None, // TODO + public_bucket_directory_hash: bucket_path.get_object_hash(), + maybe_public_bucket_prefix: bucket_path.get_optional_prefix(), + maybe_public_bucket_extension: bucket_path.get_optional_extension(), + maybe_creator_file_synthetic_id_category: IdCategory::MediaFile, // TODO: Remove this + maybe_creator_category_synthetic_id_category: IdCategory::FileUpload, // TODO: Remove this + maybe_extra_media_info: None, // TODO + is_generated_on_prem: false, // TODO + generated_by_worker: None, // TODO + generated_by_cluster: None, // TODO + maybe_mod_user_token: None, // TODO + }).await; + + match result { + Ok(result) => Ok(result.0), + Err(err) => Err(MediaFileInsertBuilderError::ProbablyDatabaseError(err)), + } + } +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs new file mode 100644 index 0000000000..f7dc467a17 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs @@ -0,0 +1,27 @@ +use std::error::Error; +use std::fmt::{Debug, Display, Formatter}; +use errors::AnyhowError; + +#[derive(Debug)] +pub enum MediaFileInsertBuilderError { + /// A user-provided field is missing (400 error). + MissingRequiredField(String), + + // TODO: We need to refactor to ensure these are actually DB errors (500s). + ProbablyDatabaseError(AnyhowError), +} + +impl Error for MediaFileInsertBuilderError {} + +impl Display for MediaFileInsertBuilderError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + MediaFileInsertBuilderError::MissingRequiredField(reason) => { + write!(f, "Missing required field: {}", reason) + } + MediaFileInsertBuilderError::ProbablyDatabaseError(reason) => { + write!(f, "Probably database error: {:?}", reason) + } + } + } +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs new file mode 100644 index 0000000000..c86603dcf7 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs @@ -0,0 +1,2 @@ +pub mod media_file_insert_builder; +mod media_file_insert_builder_error; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs index cb82ec95fb..1cd4c245d2 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs @@ -1,4 +1,5 @@ pub mod generic_insert; +pub mod insert_builder; pub mod insert_media_file_from_face_fusion; pub mod insert_media_file_from_gptsovits; pub mod insert_media_file_from_live_portrait; diff --git a/crates/lib/shared_service_components/Cargo.toml b/crates/lib/shared_service_components/Cargo.toml index 573d84ffed..72b4116d5d 100644 --- a/crates/lib/shared_service_components/Cargo.toml +++ b/crates/lib/shared_service_components/Cargo.toml @@ -27,6 +27,7 @@ errors = { workspace = true } anyhow = { workspace = true } log = { workspace = true } r2d2_redis = "0.14.0" +reqwest = { workspace = true, features = ["multipart", "gzip"] } [dev-dependencies] # None yet diff --git a/crates/lib/shared_service_components/src/lib.rs b/crates/lib/shared_service_components/src/lib.rs index 8cf4364139..c93113bb58 100644 --- a/crates/lib/shared_service_components/src/lib.rs +++ b/crates/lib/shared_service_components/src/lib.rs @@ -1,3 +1,4 @@ //! This library contains code shared between `storyteller-web` and jobs. pub mod sora_redis_credentials; +pub mod sora_redis_sentinel_refresh; diff --git a/crates/lib/shared_service_components/src/sora_redis_sentinel_refresh/mod.rs b/crates/lib/shared_service_components/src/sora_redis_sentinel_refresh/mod.rs new file mode 100644 index 0000000000..54c5907c22 --- /dev/null +++ b/crates/lib/shared_service_components/src/sora_redis_sentinel_refresh/mod.rs @@ -0,0 +1 @@ +pub mod refresh; \ No newline at end of file diff --git a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/refresh.rs b/crates/lib/shared_service_components/src/sora_redis_sentinel_refresh/refresh.rs similarity index 62% rename from crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/refresh.rs rename to crates/lib/shared_service_components/src/sora_redis_sentinel_refresh/refresh.rs index 44c4212267..2280d097d1 100644 --- a/crates/lib/api_clients/openai_sora_client/src/sentinel_refresh/refresh.rs +++ b/crates/lib/shared_service_components/src/sora_redis_sentinel_refresh/refresh.rs @@ -1,25 +1,19 @@ -// Send a POST request to env variable SORA_SENTINEL_REFRESH_URL +/// Send a POST request to env variable SORA_SENTINEL_REFRESH_URL +use errors::AnyhowResult; use reqwest::Client; use std::env; -use errors::AnyhowResult; - const SORA_SENTINEL_REFRESH_URL: &str = "http://oai-token-server/refresh_token"; - -// Refresh the Sora sentinel -// The response body does _not_ contain the new sentinel -// We should read it again from redis or generate a new one here -// For now we refresh in the server handler +/// Refresh the Sora sentinel +/// The response body does _not_ contain the new sentinel +/// We should read it again from redis or generate a new one here +/// For now we refresh in the server handler pub async fn refresh_sentinel() -> AnyhowResult { let client = Client::new(); let url = env::var("SORA_SENTINEL_REFRESH_URL").unwrap_or(SORA_SENTINEL_REFRESH_URL.to_string()); - let response = client.post(url).send().await?; - - let body = response.text().await?; - Ok(body) } diff --git a/crates/service/job/inference_job/src/job/job_types/image_generation/v2/process_single_sora_job.rs b/crates/service/job/inference_job/src/job/job_types/image_generation/v2/process_single_sora_job.rs index f32df5d93f..79667ff46f 100644 --- a/crates/service/job/inference_job/src/job/job_types/image_generation/v2/process_single_sora_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/image_generation/v2/process_single_sora_job.rs @@ -17,30 +17,30 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::state::job_dependencies::JobDependencies; use crate::util::extractors::get_polymorphic_args_from_job::get_polymorphic_args_from_job; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity; use errors::AnyhowResult; +use filesys::file_read_bytes::file_read_bytes; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use log::info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::batch_generations::insert_batch_generation_records::{insert_batch_generation_records, InsertBatchArgs}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use openai_sora_client::credentials::SoraCredentials; -use openai_sora_client::image_gen::image_gen_status::TaskStatus; -use openai_sora_client::image_gen::{wait_for_image_gen_status, Generation}; +use openai_sora_client::recipes::wait_for_image_gen_status::wait_for_image_gen_status; +use openai_sora_client::requests::image_gen::image_gen_status::{Generation, TaskStatus}; use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::redis::Commands; use r2d2_redis::RedisConnectionManager; +use shared_service_components::sora_redis_credentials::get_sora_credentials_from_redis::get_sora_credentials_from_redis; use std::collections::HashMap; use std::fs::File; use std::io::Write; use std::path::PathBuf; -use log::info; use tempdir::TempDir; -use url::Url; -use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use filesys::file_read_bytes::file_read_bytes; -use hashing::sha256::sha256_hash_file::sha256_hash_file; -use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; -use shared_service_components::sora_redis_credentials::get_sora_credentials_from_redis::get_sora_credentials_from_redis; use tokens::tokens::batch_generations::BatchGenerationToken; +use url::Url; const IMAGE_PREFIX : &str = "image_"; const IMAGE_SUFFIX : &str = ".png"; // TODO: Vary based on filetype. diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index c925dcc9d8..95260d8987 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -51,11 +51,12 @@ google_sign_in = { path = "../../../lib/api_clients/google_sign_in" } hashing = { path = "../../../lib/files/hashing" } http_server_common = { path = "../../../lib/deprecated/http_server_common" } idempotency = { path = "../../../lib/idempotency" } +images = { workspace = true } markdown = { path = "../../../lib/markdown" } media = { path = "../../../lib/files/media" } memory_caching = { path = "../../../lib/caching/memory_caching" } migration = { path = "../../../lib/migration" } -mimetypes = { path = "../../../lib/files/mimetypes" } +mimetypes = { workspace = true } mysql_queries = { path = "../../../lib/mysql_queries" } openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" } password = { path = "../../../lib/password" } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/mod.rs index 67e8e28a2e..043e3a8158 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/mod.rs @@ -1,3 +1,4 @@ pub mod upload; pub mod prompt; -pub mod get_sora_credentials_from_request; \ No newline at end of file +pub mod get_sora_credentials_from_request; +pub mod update_gpt_image_job_status_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs index a0cb3cd9e4..d2b6918f6a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs @@ -11,17 +11,17 @@ use actix_web::web::Json; use actix_web::{web, HttpRequest, HttpResponse}; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use cloud_storage::bucket_client::BucketClient; -use log::{debug, error, info}; use log::warn; +use log::{debug, error}; use mysql_queries::queries::media_files::get::get_media_file::get_media_file; use openai_sora_client::credentials::SoraCredentials; -use openai_sora_client::image_gen::SoraError; -use openai_sora_client::sentinel_refresh::generate::token::generate_token; -use openai_sora_client::sentinel_refresh::refresh_sentinel; +use openai_sora_client::requests::image_gen::SoraError; +use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token; use serde::Deserialize; use serde::Serialize; -use shared_service_components::sora_redis_credentials::set_sora_credential_field_in_redis::set_sora_credential_field_in_redis; use shared_service_components::sora_redis_credentials::keys::RedisSoraCredentialSubkey; +use shared_service_components::sora_redis_credentials::set_sora_credential_field_in_redis::set_sora_credential_field_in_redis; +use shared_service_components::sora_redis_sentinel_refresh::refresh::refresh_sentinel; use sqlx::MySqlPool; use tempdir::TempDir; use utoipa::ToSchema; @@ -37,9 +37,10 @@ use mysql_queries::payloads::generic_inference_args::generic_inference_args::{Ge use mysql_queries::payloads::generic_inference_args::inner_payloads::sora_image_gen_args::SoraImageGenArgs; use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; -use openai_sora_client::image_gen::common::{ImageSize, NumImages}; -use openai_sora_client::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; -use openai_sora_client::upload::upload_media_from_file::{sora_media_upload_from_file, SoraMediaUploadRequest}; +use openai_sora_client::creds::credential_migration::CredentialMigrationRef; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::model_weights::ModelWeightToken; @@ -83,6 +84,9 @@ pub struct EnqueueStudioImageGenRequest { pub disable_system_prompt: Option, /// Additional images to include (optional). Up to nine images. + pub additional_images: Option>, + + #[deprecated(note="use `additional_images` instead. if both are present, `additional_images` will be used.")] pub maybe_additional_images: Option>, pub maybe_number_of_samples: Option, @@ -236,13 +240,26 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, EnqueueImageGenRequestError::ServerError })?; - let mut files_to_upload = Vec::with_capacity(1 + request.maybe_additional_images.as_ref().map(|v| v.len()).unwrap_or(0)); + let additional_images; + + // NB: Frontend is calling us with the wrong field name. Let's just accept both. + if let Some(images) = &request.additional_images { + additional_images = images.clone(); + } else if let Some(images) = &request.maybe_additional_images { + additional_images = images.clone(); + } else { + additional_images = vec![]; + } + + let mut files_to_upload = Vec::with_capacity(1 + additional_images.len()); files_to_upload.push(scene_media_path.clone()); - let additional_images = request.maybe_additional_images.clone().unwrap_or_else(|| vec![]); + debug!("Additional images to download: {}", additional_images.len()); + + for (i, media_file_token ) in additional_images.iter().enumerate() { + debug!("Downloading additional image {} of {} ...", (i+1), additional_images.len()); - for media_file_token in &additional_images { let media_file_path = query_and_download_media_file(&media_file_token, &work_temp_dir, &public_bucket_client, &server_state.mysql_pool).await.map_err(|err| { error!("Failed to download additional media file: {:?}", err); EnqueueImageGenRequestError::ServerError @@ -267,11 +284,16 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len()); - for file_path in files_to_upload { - let sora_upload_response = sora_media_upload_from_file(file_path, &sora_credentials).await.map_err(|err| { - error!("Failed to upload scene media to Sora: {:?}", err); - EnqueueImageGenRequestError::ServerError - })?; + for (i, file_path) in files_to_upload.iter().enumerate() { + debug!("Uploading file {} of {} to Sora...", (i+1), files_to_upload.len()); + + let sora_upload_response = + sora_media_upload_from_file(file_path, CredentialMigrationRef::Legacy(&sora_credentials)) + .await + .map_err(|err| { + error!("Failed to upload scene media to Sora: {:?}", err); + EnqueueImageGenRequestError::ServerError + })?; debug!("Uploaded media to Sora : {:?}", sora_upload_response); sora_media_tokens.push(sora_upload_response.id); @@ -294,12 +316,14 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, let prompt = create_prompt(&request); + debug!("Sending Sora remix request with {} media tokens...", sora_media_tokens.len()); + let mut response = sora_image_gen_remix(SoraImageGenRemixRequest { prompt: prompt.clone(), num_images: NumImages::One, image_size: ImageSize::Square, sora_media_tokens: sora_media_tokens.clone(), - credentials: &sora_credentials + credentials: CredentialMigrationRef::Legacy(&sora_credentials), }).await; debug!("Sora image gen response: {:?}", response); @@ -328,7 +352,7 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, num_images: NumImages::One, image_size: ImageSize::Square, sora_media_tokens: sora_media_tokens.clone(), - credentials: &updated_sora_credentials + credentials: CredentialMigrationRef::Legacy(&updated_sora_credentials) }).await; }, Err(e) => { @@ -347,7 +371,7 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, num_images: NumImages::One, image_size: ImageSize::Square, sora_media_tokens: sora_media_tokens.clone(), - credentials: &updated_sora_credentials + credentials: CredentialMigrationRef::Legacy(&updated_sora_credentials) }).await; }, Err(e) => { @@ -485,7 +509,7 @@ async fn query_and_download_media_file(media_file_token: &MediaFileToken, downlo let download_filename = format!("download_file_{}.{}", media_file.token.as_str(), extension); let download_file_path = download_dir.path().join(download_filename); - info!("Downloading from bucket..."); + debug!("Downloading from bucket..."); bucket_client.download_file_to_disk(&media_file_bucket_path.to_full_object_pathbuf(), &download_file_path).await?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs new file mode 100644 index 0000000000..229c6f9909 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs @@ -0,0 +1,281 @@ +use std::fmt; +use std::sync::Arc; + +use crate::http_server::endpoints::engine::create_scene_handler::CreateSceneError; +use crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::GetInferenceJobStatusError; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::user_session::require_user_session::require_user_session; +use crate::state::server_state::ServerState; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::{Data, Json, Path}; +use actix_web::{web, HttpMessage, HttpRequest, HttpResponse}; +use base64::alphabet::STANDARD; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use cloud_storage::bucket_client::BucketClient; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::job_status::JobStatus; +use enums::common::job_status_plus::JobStatusPlus; +use errors::AnyhowResult; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use images::image_info::image_info::ImageInfo; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed; +use mysql_queries::queries::generic_inference::web::dismiss_finished_jobs_for_user::dismiss_finished_jobs_for_user; +use mysql_queries::queries::generic_inference::web::get_inference_job_status::{get_inference_job_status, get_inference_job_status_from_connection}; +use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus; +use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_cancelled_by_user::mark_generic_inference_job_cancelled_by_user; +use mysql_queries::queries::media_files::create::generic_insert::insert_media_file_generic::{insert_media_file_generic, InsertArgs}; +use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; +use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; +use sqlx::{MySqlConnection, MySqlPool}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; +use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_successfully_done_by_token::mark_generic_inference_job_successfully_done_by_token; + +#[derive(Deserialize, ToSchema)] +pub struct UpdateGptImageJobStatusRequest { + /// The token of the job we're updating. + pub job_token: InferenceJobToken, + + /// How to mark the job. + /// A subset of the job states that are relevant to image completion status. + pub job_status: UpdatedJobStatus, + + /// Base64-encoded image data. + /// Only present on success. + /// In the success case, there may be one or more images. + pub images: Option>, +} + +#[derive(Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum UpdatedJobStatus { + Started, + CompleteSuccess, + CompleteFailure, + AttemptFailed, + Dead, +} + +#[derive(Serialize, ToSchema)] +pub struct UpdateGptImageJobStatusSuccessResponse { + pub success: bool, +} + +#[derive(Debug, ToSchema)] +pub enum UpdateGptImageJobStatusError { + ServerError, + NotFound, + NotAuthorized, +} + +impl ResponseError for UpdateGptImageJobStatusError { + fn status_code(&self) -> StatusCode { + match *self { + UpdateGptImageJobStatusError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UpdateGptImageJobStatusError::NotFound => StatusCode::NOT_FOUND, + UpdateGptImageJobStatusError::NotAuthorized => StatusCode::UNAUTHORIZED, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + Self::ServerError => "server error".to_string(), + Self::NotFound => "not found".to_string(), + Self::NotAuthorized => "unauthorized".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for UpdateGptImageJobStatusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +/// [INTERNAL ENDPOINT] Update a GPT Image generation job status. +/// This is called by our internal infra, not users. Keep it guarded. +/// We can do secrets-based auth later. +#[utoipa::path( + post, + tag = "Jobs", + path = "/v1/image_studio/update_job_status", + responses( + (status = 200, body = UpdateGptImageJobStatusSuccessResponse), + (status = 500, body = UpdateGptImageJobStatusError), + ), + params( + ("request" = UpdateGptImageJobStatusRequest, description = "Request"), + ) +)] +pub async fn update_gpt_image_job_status_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data>) -> Result, UpdateGptImageJobStatusError> +{ + //// TODO(bt,2024-06-16): Reuse connection + //let mut mysql_connection = server_state.mysql_pool.acquire() + // .await + // .map_err(|e| { + // warn!("Could not acquire DB pool: {:?}", e); + // UpdateGptImageJobStatusError::ServerError + // })?; + + let inference_job = get_inference_job_status( + &request.job_token, + &server_state.mysql_pool + ).await; + + let inference_job = match inference_job { + Ok(Some(record)) => record, + Ok(None) => return Err(UpdateGptImageJobStatusError::NotFound), + Err(err) => { + error!("tts job query error: {:?}", err); + return Err(UpdateGptImageJobStatusError::ServerError); + } + }; + + match inference_job.status { + JobStatusPlus::CompleteSuccess => { + // Job has already been completed. Don't replay the request. + return Ok(Json(UpdateGptImageJobStatusSuccessResponse { success: true })) + }, + // TODO: Handle other states as terminal states? + //JobStatusPlus::CompleteFailure => return Err(UpdateGptImageJobStatusError::ServerError), + //JobStatusPlus::AttemptFailed => {} + //JobStatusPlus::Dead => {} + //JobStatusPlus::CancelledByUser => return Err(UpdateGptImageJobStatusError::ServerError), + //JobStatusPlus::CancelledBySystem => {} + _ => {} // Intentional fall through. + } + + if let Some(images) = &request.images { + for image in images.iter() { + let result = upload_and_save_image( + image, + &inference_job, + &server_state.mysql_pool, + &server_state.public_bucket_client, + ).await; + let media_token = match result { + Ok(media_token) => media_token, + Err(err) => { + error!("Error uploading image: {:?}", err); + return Err(UpdateGptImageJobStatusError::ServerError); + } + }; + } + } + + update_job_record(&request, &server_state, inference_job) + .await + .map_err(|err| { + error!("Error updating job: {:?}", err); + UpdateGptImageJobStatusError::ServerError + })?; + + Ok(Json(UpdateGptImageJobStatusSuccessResponse { + success: true, + })) +} + +async fn upload_and_save_image( + base64_image: &str, + inference_job: &GenericInferenceJobStatus, + mysql_pool: &MySqlPool, + public_bucket_client: &BucketClient, +) -> AnyhowResult { + + let image_bytes = BASE64_STANDARD.decode(base64_image)?; + + // Read file metadata + let file_size_bytes = image_bytes.len(); + let file_hash = sha256_hash_bytes(&image_bytes)?; + let image_info = ImageInfo::decode_image_from_bytes(&image_bytes)?; + let mimetype = image_info.mime_type(); + + // Upload file + const PREFIX : Option<&str> = Some("image_"); + + let extension_with_period = image_info.file_extension() + .map(|ext| ext.extension_with_period()) + .unwrap_or_else(|| ".png"); + + let media_file_type = match mimetype { + "image/png" => MediaFileType::Png, + "image/jpeg" => MediaFileType::Jpg, + _ => MediaFileType::Png, + }; + + let bucket_upload_path = MediaFileBucketPath::generate_new( + PREFIX, Some(extension_with_period)); + + info!("Uploading media to bucket path: {}", bucket_upload_path.get_full_object_path_str()); + + public_bucket_client.upload_file_with_content_type( + bucket_upload_path.get_full_object_path_str(), + image_bytes.as_ref(), + mimetype + ).await?; + + let media_token = MediaFileInsertBuilder::new() + .maybe_creator_user(inference_job.user_details.maybe_creator_user_token.as_ref()) + .maybe_creator_anonymous_visitor(inference_job.user_details.maybe_creator_anonymous_visitor_token.as_ref()) + .creator_ip_address(&inference_job.user_details.creator_ip_address) + .public_bucket_directory_hash(&bucket_upload_path) + .media_file_class(MediaFileClass::Image) + .media_file_type(media_file_type) + .media_file_origin_category(MediaFileOriginCategory::Inference) + .media_file_origin_product_category(MediaFileOriginProductCategory::ImageStudio) + .mime_type(mimetype) + .file_size_bytes(file_size_bytes as u64) + .frame_width(image_info.width()) + .frame_height(image_info.height()) + .checksum_sha2(&file_hash) + .insert_pool(mysql_pool) + .await?; + + Ok(media_token) +} + +async fn update_job_record( + request: &Json, + server_state: &Data>, + inference_job: GenericInferenceJobStatus +) -> AnyhowResult<()> { + + match inference_job.status { + JobStatusPlus::Pending => {} + JobStatusPlus::Started => {} + JobStatusPlus::CompleteSuccess => { + mark_generic_inference_job_successfully_done_by_token( + &server_state.mysql_pool, + &request.job_token, + None, + None, + None, + None, + ).await?; + } + // TODO: Handle terminal states + JobStatusPlus::CompleteFailure => {} + JobStatusPlus::AttemptFailed => {} + JobStatusPlus::Dead => {} + JobStatusPlus::CancelledByUser => {} + JobStatusPlus::CancelledBySystem => {} + } + Ok(()) +} diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_image_studio_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_image_studio_routes.rs index caf25ae473..65abd6a967 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_image_studio_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_image_studio_routes.rs @@ -1,4 +1,5 @@ use crate::http_server::endpoints::image_studio::prompt::enqueue_studio_image_generation_handler::enqueue_studio_image_generation_handler; +use crate::http_server::endpoints::image_studio::update_gpt_image_job_status_handler::update_gpt_image_job_status_handler; use crate::http_server::endpoints::image_studio::upload::upload_snapshot_media_file_handler::upload_snapshot_media_file_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; @@ -25,5 +26,9 @@ where .route(web::post().to(enqueue_studio_image_generation_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/update_job_status") + .route(web::post().to(update_gpt_image_job_status_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) } diff --git a/frontend/apps/artcraft-website/index.html b/frontend/apps/artcraft-website/index.html new file mode 100644 index 0000000000..a383a1310f --- /dev/null +++ b/frontend/apps/artcraft-website/index.html @@ -0,0 +1,22 @@ + + + + + ArtCraft + + + + + + + + + + +
+ + + diff --git a/frontend/apps/artcraft-website/package.json b/frontend/apps/artcraft-website/package.json new file mode 100644 index 0000000000..84e2b409e5 --- /dev/null +++ b/frontend/apps/artcraft-website/package.json @@ -0,0 +1,5 @@ +{ + "name": "@frontend/artcraft-website", + "version": "0.0.1", + "private": true +} diff --git a/frontend/apps/artcraft-website/postcss.config.js b/frontend/apps/artcraft-website/postcss.config.js new file mode 100644 index 0000000000..5a8ffc4899 --- /dev/null +++ b/frontend/apps/artcraft-website/postcss.config.js @@ -0,0 +1,15 @@ +const { join } = require('path'); + +// Note: If you use library-specific PostCSS/Tailwind configuration then you should remove the `postcssConfig` build +// option from your application's configuration (i.e. project.json). +// +// See: https://nx.dev/guides/using-tailwind-css-in-react#step-4:-applying-configuration-to-libraries + +module.exports = { + plugins: { + tailwindcss: { + config: join(__dirname, 'tailwind.config.js'), + }, + autoprefixer: {}, + }, +} diff --git a/frontend/apps/artcraft-website/public/favicon.ico b/frontend/apps/artcraft-website/public/favicon.ico new file mode 100644 index 0000000000..df243d83f7 Binary files /dev/null and b/frontend/apps/artcraft-website/public/favicon.ico differ diff --git a/frontend/apps/artcraft-website/public/images/3d-interface-preview.jpg b/frontend/apps/artcraft-website/public/images/3d-interface-preview.jpg new file mode 100644 index 0000000000..e6a5791961 Binary files /dev/null and b/frontend/apps/artcraft-website/public/images/3d-interface-preview.jpg differ diff --git a/frontend/apps/artcraft-website/public/images/artcraft-logo.png b/frontend/apps/artcraft-website/public/images/artcraft-logo.png new file mode 100644 index 0000000000..8cf2cab819 Binary files /dev/null and b/frontend/apps/artcraft-website/public/images/artcraft-logo.png differ diff --git a/frontend/apps/artcraft-website/public/videos/artcraft-3d-demo.mp4 b/frontend/apps/artcraft-website/public/videos/artcraft-3d-demo.mp4 new file mode 100644 index 0000000000..3ceb1656a9 Binary files /dev/null and b/frontend/apps/artcraft-website/public/videos/artcraft-3d-demo.mp4 differ diff --git a/frontend/apps/artcraft-website/public/videos/artcraft-canvas-demo.mp4 b/frontend/apps/artcraft-website/public/videos/artcraft-canvas-demo.mp4 new file mode 100644 index 0000000000..a910aa6144 Binary files /dev/null and b/frontend/apps/artcraft-website/public/videos/artcraft-canvas-demo.mp4 differ diff --git a/frontend/apps/artcraft-website/src/app/app.module.css b/frontend/apps/artcraft-website/src/app/app.module.css new file mode 100644 index 0000000000..7b88fbabf8 --- /dev/null +++ b/frontend/apps/artcraft-website/src/app/app.module.css @@ -0,0 +1 @@ +/* Your styles goes here. */ diff --git a/frontend/apps/artcraft-website/src/app/app.spec.tsx b/frontend/apps/artcraft-website/src/app/app.spec.tsx new file mode 100644 index 0000000000..c19e8c1d14 --- /dev/null +++ b/frontend/apps/artcraft-website/src/app/app.spec.tsx @@ -0,0 +1,16 @@ +import { render } from '@testing-library/react'; +import { BrowserRouter } from 'react-router-dom'; + +import App from './app'; + +describe('App', () => { + it('should render successfully', () => { + const { baseElement } = render(); + expect(baseElement).toBeTruthy(); + }); + + it('should have a greeting as the title', () => { + const { getAllByText } = render(); + expect(getAllByText(new RegExp('Welcome @frontend/artcraft-website', 'gi')).length > 0).toBeTruthy(); + }); +}); diff --git a/frontend/apps/artcraft-website/src/app/app.tsx b/frontend/apps/artcraft-website/src/app/app.tsx new file mode 100644 index 0000000000..88449a245f --- /dev/null +++ b/frontend/apps/artcraft-website/src/app/app.tsx @@ -0,0 +1,22 @@ +import { Route, Routes } from "react-router-dom"; +import Landing from "../pages/landing"; +import Download from "../pages/download"; +import Navbar from "../components/navbar"; +import Footer from "../components/footer"; + +export function App() { + return ( +
+ + + + } /> + } /> + + +
+
+ ); +} + +export default App; diff --git a/frontend/apps/artcraft-website/src/assets/.gitkeep b/frontend/apps/artcraft-website/src/assets/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frontend/apps/artcraft-website/src/components/download-button/download-button.tsx b/frontend/apps/artcraft-website/src/components/download-button/download-button.tsx new file mode 100644 index 0000000000..c0d1ccbf2a --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/download-button/download-button.tsx @@ -0,0 +1,58 @@ +import { Button } from "@storyteller/ui-button"; +import { + faApple, + faWindows, + faLinux, +} from "@fortawesome/free-brands-svg-icons"; +import { isMobile, isWindows, isMacOs } from "react-device-detect"; +import { DOWNLOAD_LINKS } from "../../config/downloads"; + +interface DownloadButtonProps { + className?: string; +} + +export const DownloadButton = ({ className = "" }: DownloadButtonProps) => { + // Helper function to detect Linux + const isLinux = + !isWindows && + !isMacOs && + navigator.platform.toLowerCase().includes("linux"); + + const getDownloadLink = () => { + if (isWindows) return DOWNLOAD_LINKS.WINDOWS; + if (isMacOs) return DOWNLOAD_LINKS.MACOS; + if (isLinux) return DOWNLOAD_LINKS.LINUX; + return null; + }; + + const getIcon = () => { + if (isMobile) return undefined; + if (isWindows) return faWindows; + if (isMacOs) return faApple; + if (isLinux) return faLinux; + return undefined; + }; + + const downloadLink = getDownloadLink(); + + return ( + + ); +}; diff --git a/frontend/apps/artcraft-website/src/components/download-button/index.ts b/frontend/apps/artcraft-website/src/components/download-button/index.ts new file mode 100644 index 0000000000..2605d8dabd --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/download-button/index.ts @@ -0,0 +1 @@ +export * from "./download-button"; diff --git a/frontend/apps/artcraft-website/src/components/footer/footer.tsx b/frontend/apps/artcraft-website/src/components/footer/footer.tsx new file mode 100644 index 0000000000..04d1d93f64 --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/footer/footer.tsx @@ -0,0 +1,79 @@ +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { + faTiktok, + faDiscord, + faYoutube, +} from "@fortawesome/free-brands-svg-icons"; +import { SOCIAL_LINKS } from "../../config/links"; + +const navigation = { + main: [ + { name: "Home", href: "/" }, + { name: "Download", href: "/download" }, + ], + social: [ + { + name: "Discord", + href: SOCIAL_LINKS.DISCORD, + icon: (props: any) => , + }, + { + name: "YouTube", + href: SOCIAL_LINKS.YOUTUBE, + icon: (props: any) => , + }, + { + name: "TikTok", + href: SOCIAL_LINKS.TIKTOK, + icon: (props: any) => , + }, + ], +}; + +export default function Example() { + return ( +
+
+ {/* */} + {/* */} +
+
+ {navigation.social.map((item) => ( + + {item.name} + + ))} +
+

+ © 2025 ArtCraft. All rights reserved. +

+
+
+
+ ); +} diff --git a/frontend/apps/artcraft-website/src/components/footer/index.ts b/frontend/apps/artcraft-website/src/components/footer/index.ts new file mode 100644 index 0000000000..7614ec3686 --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/footer/index.ts @@ -0,0 +1 @@ +export { default } from "./footer"; diff --git a/frontend/apps/artcraft-website/src/components/index.ts b/frontend/apps/artcraft-website/src/components/index.ts new file mode 100644 index 0000000000..c152451d62 --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/index.ts @@ -0,0 +1,3 @@ +export * from "./navbar"; +export * from "./footer"; +export * from "./download-button"; diff --git a/frontend/apps/artcraft-website/src/components/navbar/index.ts b/frontend/apps/artcraft-website/src/components/navbar/index.ts new file mode 100644 index 0000000000..8fb3806210 --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/navbar/index.ts @@ -0,0 +1 @@ +export { default } from "./navbar"; diff --git a/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx b/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx new file mode 100644 index 0000000000..603120afca --- /dev/null +++ b/frontend/apps/artcraft-website/src/components/navbar/navbar.tsx @@ -0,0 +1,107 @@ +import { + Disclosure, + DisclosureButton, + DisclosurePanel, +} from "@headlessui/react"; +import { faBars } from "@fortawesome/pro-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Button } from "@storyteller/ui-button"; +import { twMerge } from "tailwind-merge"; +import { useEffect, useState } from "react"; + +const navigation = [ + { name: "Home", href: "/", current: true }, + { name: "Download", href: "/download", current: false }, +]; + +export default function Navbar() { + const [scrolled, setScrolled] = useState(false); + + useEffect(() => { + const handleScroll = () => { + setScrolled(window.scrollY > 0); + }; + + window.addEventListener("scroll", handleScroll); + return () => window.removeEventListener("scroll", handleScroll); + }, []); + + return ( + +
+
+
+
+ + ArtCraft + +
+
+ {navigation.map((item) => ( + + {item.name} + + ))} +
+
+
+
+ +
+
+ {/* Mobile menu button */} + + + Open main menu + + +
+
+
+
+ + +
+ {navigation.map((item) => ( + + {item.name} + + ))} +
+ + + ); +} diff --git a/frontend/apps/artcraft-website/src/config/downloads.ts b/frontend/apps/artcraft-website/src/config/downloads.ts new file mode 100644 index 0000000000..555fec36c6 --- /dev/null +++ b/frontend/apps/artcraft-website/src/config/downloads.ts @@ -0,0 +1,5 @@ +export const DOWNLOAD_LINKS = { + WINDOWS: "https://getartcraft.com/", // Replace with actual Windows download link + MACOS: "https://getartcraft.com/", // Replace with actual MacOS download link + LINUX: "https://getartcraft.com/", // Replace with actual Linux download link +} as const; diff --git a/frontend/apps/artcraft-website/src/config/links.ts b/frontend/apps/artcraft-website/src/config/links.ts new file mode 100644 index 0000000000..ded8fa8211 --- /dev/null +++ b/frontend/apps/artcraft-website/src/config/links.ts @@ -0,0 +1,5 @@ +export const SOCIAL_LINKS = { + DISCORD: "https://discord.gg/75svZP2Vje", + YOUTUBE: "https://www.youtube.com/@OfficialArtCraftStudios", + TIKTOK: "https://www.tiktok.com/@artcraft.studios", +} as const; diff --git a/frontend/apps/artcraft-website/src/main.tsx b/frontend/apps/artcraft-website/src/main.tsx new file mode 100644 index 0000000000..1195deb0c3 --- /dev/null +++ b/frontend/apps/artcraft-website/src/main.tsx @@ -0,0 +1,16 @@ +import { StrictMode } from 'react'; +import { BrowserRouter } from 'react-router-dom'; +import * as ReactDOM from 'react-dom/client'; +import App from './app/app'; + +const root = ReactDOM.createRoot( + document.getElementById('root') as HTMLElement +); + +root.render( + + + + + +); \ No newline at end of file diff --git a/frontend/apps/artcraft-website/src/pages/download/download.tsx b/frontend/apps/artcraft-website/src/pages/download/download.tsx new file mode 100644 index 0000000000..e21ed8c470 --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/download/download.tsx @@ -0,0 +1,119 @@ +import { + faApple, + faWindows, + faLinux, +} from "@fortawesome/free-brands-svg-icons"; +import { faArrowDownToLine } from "@fortawesome/pro-solid-svg-icons"; +import { DownloadButton } from "../../components/download-button"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { Button } from "@storyteller/ui-button"; +import { DOWNLOAD_LINKS } from "../../config/downloads"; + +const Download = () => { + const systemRequirements = [ + { + os: "Windows", + icon: faWindows, + requirements: [ + "Windows 10 (64-bit) or newer", + "8GB RAM minimum", + "4GB available storage", + ], + }, + { + os: "macOS", + icon: faApple, + requirements: [ + "macOS 12.0 or newer", + "8GB RAM minimum", + "4GB available storage", + ], + }, + { + os: "Linux", + icon: faLinux, + requirements: [ + "Ubuntu 20.04 or newer", + "8GB RAM minimum", + "4GB available storage", + ], + }, + ]; + + return ( +
+
+ + {/* Hero Section */} +
+
+

+ Download ArtCraft +

+ +

+ Use ArtCraft to easily create AI-powered artwork on your computer, + with canvas editing and 3D scene composition. +

+ +
+ +
+ +
+ ArtCraft Interface Preview +
+ + {/* System Requirements */} +
+

System Requirements

+
+ {systemRequirements.map((system, index) => ( +
+
+ +

{system.os}

+
+
    + {system.requirements.map((req, idx) => ( +
  • + {req} +
    • + ))} +
+ +
+ ))} +
+
+
+
+
+ ); +}; + +export default Download; diff --git a/frontend/apps/artcraft-website/src/pages/download/index.ts b/frontend/apps/artcraft-website/src/pages/download/index.ts new file mode 100644 index 0000000000..0ed4d3e7df --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/download/index.ts @@ -0,0 +1 @@ +export { default } from "./download"; diff --git a/frontend/apps/artcraft-website/src/pages/index.ts b/frontend/apps/artcraft-website/src/pages/index.ts new file mode 100644 index 0000000000..ebe1ac77e5 --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/index.ts @@ -0,0 +1,2 @@ +export * from "./landing"; +export * from "./download"; diff --git a/frontend/apps/artcraft-website/src/pages/landing/index.ts b/frontend/apps/artcraft-website/src/pages/landing/index.ts new file mode 100644 index 0000000000..44201cad15 --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/landing/index.ts @@ -0,0 +1 @@ +export { default } from "./landing"; diff --git a/frontend/apps/artcraft-website/src/pages/landing/landing.tsx b/frontend/apps/artcraft-website/src/pages/landing/landing.tsx new file mode 100644 index 0000000000..0466ad2187 --- /dev/null +++ b/frontend/apps/artcraft-website/src/pages/landing/landing.tsx @@ -0,0 +1,201 @@ +import { Button } from "@storyteller/ui-button"; +import { DownloadButton } from "../../components/download-button"; +import { faArrowDownToLine } from "@fortawesome/pro-solid-svg-icons"; +import { faDiscord } from "@fortawesome/free-brands-svg-icons"; +import { SOCIAL_LINKS } from "../../config/links"; + +const Landing = () => { + const videos = [ + { + src: "https://www.youtube.com/embed/pGn-1BKo3nY?si=q4dm0ICb6wF1JW8N", + }, + { + src: "https://www.youtube.com/embed/_FkKf7sECk4?si=Jx38ZGSoyYj0hObr", + }, + { + src: "https://www.youtube.com/embed/7x7IZkHiGD8?si=tL8nK4CULigpfHQR", + }, + ]; + + return ( +
+
+ {/* Hero Section */} +
+
+
+
+

+ AI Canvas & Scene Editor +
+ On your + + local machine + + . +

+
+ +

+ Create stunning artwork with the power of AI. Perfect for artists, + designers, and creators who want to push the boundaries of their + creativity. +

+ +
+ + +
+ +
+
+
+ + {/* Video Sections */} +
+ {/* First Video Section */} +
+
+ +
+
+
+

+ Draw it, Dream it, AI it +

+

+ Draw freely, arrange images, and create collages on our + intuitive 2D canvas. Once your composition is ready, use AI to + generate a polished artwork that matches your exact layout and + vision. Perfect for sketching ideas and turning them into + stunning finished pieces. +

+
+ +
+
+ + {/* Second Video Section */} +
+
+
+

+ Compose in 3D, Bring your ideas to life +

+

+ Compose your perfect 3D scene by arranging elements and + positioning your camera. Our AI understands your scene layout + and generates consistent, high-quality 3D artwork that matches + your composition, including maintaining character consistency + throughout your scenes. +

+
+ +
+
+ +
+
+
+ +
+ {/* Videos Section */} +
+
+

+ Made using{" "} + + ArtCraft + +

+

+ See examples of artwork created using ArtCraft. +

+
+
+ {videos.map((video, index) => ( +
+
+