From e9fc9cd40e7b1889bc8de2ffd2143b13d04737a0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 3 Oct 2025 18:38:12 -0400 Subject: [PATCH] generate sora2 in artcraft (wip) --- ...ate_sora2_video_with_session_auto_renew.rs | 53 +++++++++++++ .../src/recipes/generate_sora2_video/mod.rs | 1 + ...st_sora2_drafts_with_session_auto_renew.rs | 45 +++++++++++ .../src/recipes/list_sora2_drafts/mod.rs | 1 + .../recipes/maybe_upgrade_or_renew_session.rs | 1 + .../openai_sora_client/src/recipes/mod.rs | 3 + .../recipes/utils/maybe_renew_session_jwt.rs | 48 ++++++++++++ .../src/recipes/utils/mod.rs | 1 + .../generate_sora2_video.rs | 1 + .../src/requests/generate_sora2_video/mod.rs | 2 +- .../list_sora2_drafts/http_response.rs | 20 +++-- .../list_sora2_drafts/list_sora2_drafts.rs | 78 ++++++++++++++----- .../enqueue_image_to_video_command.rs | 41 +++++++--- .../generic/handle_video_artcraft.rs | 6 ++ .../commands/enqueue/image_to_video/mod.rs | 3 +- .../sora2/handle_sora2_video.rs | 49 ++++++++++++ .../sora2/handle_sora2_video_sora.rs | 73 +++++++++++++++++ .../enqueue/image_to_video/sora2/mod.rs | 2 + .../commands/enqueue/task_enqueue_success.rs | 1 + .../core/events/generation_events/common.rs | 2 + .../commands/check_sora_session_command.rs | 4 +- .../handle_failed_generations.rs | 2 +- .../handle_successful_generations.rs | 2 +- .../sora_task_polling_thread.rs | 6 +- .../enums/src/tauri/tasks/task_model_type.rs | 11 ++- 25 files changed, 407 insertions(+), 49 deletions(-) create mode 100644 crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/generate_sora2_video_with_session_auto_renew.rs create mode 100644 crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/mod.rs create mode 100644 crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/list_sora2_drafts_with_session_auto_renew.rs create mode 100644 crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/mod.rs create mode 100644 crates/api_clients/openai_sora_client/src/recipes/utils/maybe_renew_session_jwt.rs create mode 100644 crates/api_clients/openai_sora_client/src/recipes/utils/mod.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video_sora.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/mod.rs diff --git a/crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/generate_sora2_video_with_session_auto_renew.rs b/crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/generate_sora2_video_with_session_auto_renew.rs new file mode 100644 index 0000000000..0efe308dce --- /dev/null +++ b/crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/generate_sora2_video_with_session_auto_renew.rs @@ -0,0 +1,53 @@ +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::error::sora_error::SoraError; +use crate::recipes::utils::maybe_renew_session_jwt::maybe_renew_session_jwt; +use crate::requests::generate_sora2_video::generate_sora2_video::{generate_sora2_video, GenerateSora2VideoArgs, GenerateSora2VideoResponse}; + +/// Generate Sora 2 video with session auto-renewal. +/// If a new sora credential is returned, replace the old one with the new one. +pub async fn generate_sora2_video_with_session_auto_renew( + args: GenerateSora2VideoArgs<'_>, +) -> Result<(GenerateSora2VideoResponse, Option), SoraError> { + let mut maybe_new_creds = maybe_renew_session_jwt(&args.credentials).await?; + + let use_creds = maybe_new_creds.as_ref() + .unwrap_or_else(|| &args.credentials); + + let mut request = args.clone(); + request.credentials = use_creds; + + let result = generate_sora2_video(request).await; + + match result { + Ok(response) => Ok((response, maybe_new_creds)), + Err(err) => Err(err), // TODO: Should we retry? + } +} + +#[cfg(test)] +mod tests { + use crate::recipes::generate_sora2_video::generate_sora2_video_with_session_auto_renew::generate_sora2_video_with_session_auto_renew; + use crate::requests::generate_sora2_video::generate_sora2_video::GenerateSora2VideoArgs; + use crate::test_utils::get_test_credentials::get_test_credentials; + use errors::AnyhowResult; + + #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. + #[tokio::test] + pub async fn manual_test() -> AnyhowResult<()> { + let creds = get_test_credentials()?; + + let args = GenerateSora2VideoArgs { + prompt: "a dog playing tennis", + credentials: &creds, + request_timeout: None, + }; + + let (result , creds) = + generate_sora2_video_with_session_auto_renew(args).await?; + + println!("result: {:#?}", result); + assert_eq!(1, 2); + + Ok(()) + } +} diff --git a/crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/mod.rs b/crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/mod.rs new file mode 100644 index 0000000000..300e294aaa --- /dev/null +++ b/crates/api_clients/openai_sora_client/src/recipes/generate_sora2_video/mod.rs @@ -0,0 +1 @@ +pub mod generate_sora2_video_with_session_auto_renew; \ No newline at end of file diff --git a/crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/list_sora2_drafts_with_session_auto_renew.rs b/crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/list_sora2_drafts_with_session_auto_renew.rs new file mode 100644 index 0000000000..021de7e6a3 --- /dev/null +++ b/crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/list_sora2_drafts_with_session_auto_renew.rs @@ -0,0 +1,45 @@ +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::error::sora_error::SoraError; +use crate::recipes::utils::maybe_renew_session_jwt::maybe_renew_session_jwt; +use crate::requests::list_sora2_drafts::list_sora2_drafts::{list_sora2_drafts, ListSora2DraftsResponse}; + +/// Check Sora 2 drafts with session auto-renewal. +/// If a new sora credential is returned, replace the old one with the new one. +pub async fn list_sora2_drafts_with_session_auto_renew( + credentials: &SoraCredentialSet, +) -> Result<(ListSora2DraftsResponse, Option), SoraError> { + + let mut maybe_new_creds = maybe_renew_session_jwt(&credentials).await?; + + let use_creds = maybe_new_creds.as_ref() + .unwrap_or_else(|| &credentials); + + let result = list_sora2_drafts(&use_creds).await; + + match result { + Ok(response) => Ok((response, maybe_new_creds)), + Err(err) => Err(err), // TODO: Should we retry? + } +} + + +#[cfg(test)] +mod tests { + use crate::recipes::list_sora2_drafts::list_sora2_drafts_with_session_auto_renew::list_sora2_drafts_with_session_auto_renew; + use crate::test_utils::get_test_credentials::get_test_credentials; + use errors::AnyhowResult; + + #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. + #[tokio::test] + pub async fn manual_test() -> AnyhowResult<()> { + let creds = get_test_credentials()?; + + let (result , creds) = + list_sora2_drafts_with_session_auto_renew(&creds).await?; + + println!("result: {:#?}", result); + assert_eq!(1, 2); + + Ok(()) + } +} diff --git a/crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/mod.rs b/crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/mod.rs new file mode 100644 index 0000000000..4909b4b23b --- /dev/null +++ b/crates/api_clients/openai_sora_client/src/recipes/list_sora2_drafts/mod.rs @@ -0,0 +1 @@ +pub mod list_sora2_drafts_with_session_auto_renew; \ No newline at end of file diff --git a/crates/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs b/crates/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs index 071b8db11a..1f179791d9 100644 --- a/crates/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs +++ b/crates/api_clients/openai_sora_client/src/recipes/maybe_upgrade_or_renew_session.rs @@ -14,6 +14,7 @@ const EXPIRATION_DEADLINE : TimeDelta = TimeDelta::hours(12); /// Call this at session startup to generate the tokens for the first time. /// Call this periodically to refresh the JWT as it nears its expiration date. /// This will not refresh the sentinel token once it's been generated - we can use other flows to accomplish that. +#[deprecated(note="The sentinel appears to be going away. Use `maybe_renew_session_jwt` instead for newer '2.0' code.")] pub async fn maybe_upgrade_or_renew_session(sora_credentials: &mut SoraCredentialSet) -> Result { let mut refresh_jwt = !sora_credentials.jwt_bearer_token.is_some(); let refresh_sentinel = !sora_credentials.sora_sentinel.is_some(); diff --git a/crates/api_clients/openai_sora_client/src/recipes/mod.rs b/crates/api_clients/openai_sora_client/src/recipes/mod.rs index 6cecc4e81b..98675c8669 100644 --- a/crates/api_clients/openai_sora_client/src/recipes/mod.rs +++ b/crates/api_clients/openai_sora_client/src/recipes/mod.rs @@ -1,8 +1,11 @@ +pub mod generate_sora2_video; pub mod image_remix_with_session_auto_renew; pub mod image_upload_from_bytes_with_session_auto_renew; pub mod image_upload_from_file_with_session_auto_renew; pub mod list_classic_sora_tasks_with_session_auto_renew; +pub mod list_sora2_drafts; pub mod maybe_refresh_credentials_on_sora_error; pub mod maybe_upgrade_or_renew_session; pub mod simple_image_gen_with_session_auto_renew; +pub mod utils; pub mod wait_for_image_gen_status; \ No newline at end of file diff --git a/crates/api_clients/openai_sora_client/src/recipes/utils/maybe_renew_session_jwt.rs b/crates/api_clients/openai_sora_client/src/recipes/utils/maybe_renew_session_jwt.rs new file mode 100644 index 0000000000..a663c726e7 --- /dev/null +++ b/crates/api_clients/openai_sora_client/src/recipes/utils/maybe_renew_session_jwt.rs @@ -0,0 +1,48 @@ +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use crate::error::sora_error::SoraError; +use crate::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use crate::requests::auth_bearer::generate_bearer_jwt_with_cookie::generate_bearer_jwt_with_cookie; +use chrono::{TimeDelta, Utc}; +use log::{info, warn}; +use std::ops::Sub; + +const EXPIRATION_DEADLINE : TimeDelta = TimeDelta::hours(12); + +/// Returns a new set of credentials if they were updated, or None if no update was needed. +pub async fn maybe_renew_session_jwt(creds: &SoraCredentialSet) -> Result, SoraError> { + let mut refresh_jwt = false; + + match creds.jwt_bearer_token.as_ref() { + None => { + info!("JWT not set."); + refresh_jwt = true; + } + Some(jwt) => { + let now = Utc::now(); + let refresh_deadline = now.sub(EXPIRATION_DEADLINE); + if jwt.jwt_claims().expiration.lt(&now) { + info!("JWT expiration time has elapsed."); + refresh_jwt = true; + } else if jwt.jwt_claims().expiration.lt(&refresh_deadline) { + info!("JWT expiration time is under the expiration deadline."); + refresh_jwt = true; + } + } + } + + if !refresh_jwt { + return Ok(None); + } + + info!("Refreshing JWT..."); + let token = generate_bearer_jwt_with_cookie(creds.cookies.as_str()).await?; + + info!("Parsing JWT bearer token..."); + let token = SoraJwtBearerToken::new(token)?; + + let mut updated_creds = creds.clone(); + updated_creds.jwt_bearer_token = Some(token); + + Ok(Some(updated_creds)) +} diff --git a/crates/api_clients/openai_sora_client/src/recipes/utils/mod.rs b/crates/api_clients/openai_sora_client/src/recipes/utils/mod.rs new file mode 100644 index 0000000000..dfb928a112 --- /dev/null +++ b/crates/api_clients/openai_sora_client/src/recipes/utils/mod.rs @@ -0,0 +1 @@ +pub mod maybe_renew_session_jwt; \ No newline at end of file diff --git a/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/generate_sora2_video.rs b/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/generate_sora2_video.rs index 3c2248d4ab..c635fa749d 100644 --- a/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/generate_sora2_video.rs +++ b/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/generate_sora2_video.rs @@ -14,6 +14,7 @@ use wreq::Client; const GENERATE_SORA_2_VIDEO_URL: &str = "https://sora.chatgpt.com/backend/nf/create"; +#[derive(Clone)] pub struct GenerateSora2VideoArgs<'a> { pub prompt: &'a str, pub credentials: &'a SoraCredentialSet, diff --git a/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/mod.rs b/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/mod.rs index 2346c62ee1..34de76d9a8 100644 --- a/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/mod.rs +++ b/crates/api_clients/openai_sora_client/src/requests/generate_sora2_video/mod.rs @@ -1,3 +1,3 @@ -mod generate_sora2_video; +pub mod generate_sora2_video; mod http_request; mod http_response; \ No newline at end of file diff --git a/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/http_response.rs b/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/http_response.rs index 98ba49dc67..5c319923ff 100644 --- a/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/http_response.rs +++ b/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/http_response.rs @@ -11,22 +11,28 @@ pub (super) struct HttpDraftsResponse { #[derive(Deserialize)] pub (super) struct PartialItem { - // eg. "gen_01abc..." + /// The generation id, eg. "gen_01abc..." pub id: String, + /// The parent task id, eg "task_01abc..." pub task_id: String, //pub generation_id: String, // it looks like this matches "id" for now + /// The kind of draft + pub kind: DraftKind, + // The text prompt pub prompt: String, - // URL to the generation - pub url: String, + /// URL to the generation + /// This will not be present if the request failed, eg. for content reasons. + pub url: Option, - // "downloadable" URL to the generation - // it looks like this matches "url" for now - pub downloadable_url: String, + /// "downloadable" URL to the generation + /// it looks like this matches "url" for now + /// This will not be present if the request failed, eg. for content reasons. + pub downloadable_url: Option, /// If there was a content violation, this will describe why. pub reason_str: Option, @@ -38,7 +44,7 @@ pub (super) struct PartialItem { #[derive(Deserialize)] #[serde(rename_all = "snake_case")] pub enum DraftKind { - /// Video + /// Generated video SoraDraft, /// Video was blocked due to content diff --git a/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/list_sora2_drafts.rs b/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/list_sora2_drafts.rs index eedc9e86ec..859bf2db3f 100644 --- a/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/list_sora2_drafts.rs +++ b/crates/api_clients/openai_sora_client/src/requests/list_sora2_drafts/list_sora2_drafts.rs @@ -5,7 +5,7 @@ use crate::error::sora_error::SoraError; use crate::error::sora_generic_api_error::SoraGenericApiError; use crate::requests::common::task_id::TaskId; use crate::requests::deprecated::job_status::sora_job_status::{Generation, StatusRequest, VideoGenStatusResponse}; -use crate::requests::list_sora2_drafts::http_response::HttpDraftsResponse; +use crate::requests::list_sora2_drafts::http_response::{DraftKind, HttpDraftsResponse}; use crate::utils_internal::classify_general_http_status_code_and_body::classify_general_http_status_code_and_body; use log::{debug, error, info, warn}; use serde_derive::Deserialize; @@ -23,13 +23,24 @@ pub struct ListSora2DraftsResponse { } #[derive(Clone, Debug)] -pub struct Draft { +pub enum Draft { + Success(DraftSuccess), + Failure(DraftFailure), +} + +#[derive(Clone, Debug)] +pub struct DraftSuccess { pub id: String, pub task_id: TaskId, pub url: String, pub prompt: String, } +#[derive(Clone, Debug)] +pub struct DraftFailure { + pub task_id: TaskId, + pub reason_message: String, +} pub async fn list_sora2_drafts(credentials: &SoraCredentialSet) -> Result { @@ -85,9 +96,6 @@ pub async fn list_sora2_drafts(credentials: &SoraCredentialSet) -> Result Result { + match draft.url { + Some(url) => { + // Success case + Draft::Success(DraftSuccess { + id: draft.id, + task_id, + url, + prompt: draft.prompt, + }) + } + None => { + return Draft::Failure(DraftFailure { + task_id, + reason_message: "Generation failed. URL missing from the draft.".to_string(), + }); + } + } + } + DraftKind::SoraContentViolation => { + Draft::Failure(DraftFailure { + task_id, + reason_message: format!( + "Generation failed due to content violation: {}", + draft.reason_str + .unwrap_or_else(|| "no additional reason".to_string())), + }) + } + DraftKind::Unknown(value) => { + Draft::Failure(DraftFailure { + task_id, + reason_message: format!( + "Generation failed with unknown failure code: {}. Possible message: {}", + value, + draft.reason_str + .unwrap_or_else(|| "no additional message".to_string())), + }) + } + } + }).collect(), }) } -/* -Error: SoraGenericApiError: UncategorizedBadResponseWithStatusAndBody { status_code: 400, -body: "{\n \"error\": {\n \"message\": \" -[{'type': 'missing', 'loc': ('body',), 'msg': 'Field required', 'input': None}]\",\n -\"type\": \"invalid_request_error\",\n \"param\": null,\n \"code\": null\n }\n}" } - - */ - #[cfg(test)] mod tests { - use crate::requests::list_classic_tasks::list_classic_tasks::list_classic_tasks; + use crate::requests::list_sora2_drafts::list_sora2_drafts::list_sora2_drafts; use crate::test_utils::get_test_credentials::get_test_credentials; use errors::AnyhowResult; - use crate::requests::list_sora2_drafts::list_sora2_drafts::list_sora2_drafts; #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. #[tokio::test] diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs index f747dfdc71..a5fed5d919 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/enqueue_image_to_video_command.rs @@ -1,6 +1,7 @@ use crate::core::commands::enqueue::common::notify_frontend_of_errors::notify_frontend_of_errors; use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError, MissingCredentialsReason, ProviderFailureReason}; use crate::core::commands::enqueue::image_to_video::generic::handle_video::handle_video; +use crate::core::commands::enqueue::image_to_video::sora2::handle_sora2_video::handle_sora2_video; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::commands::response::shorthand::Response; @@ -14,6 +15,7 @@ use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; use crate::core::state::task_database::TaskDatabase; use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; @@ -39,6 +41,9 @@ pub enum VideoModel { #[serde(rename = "seedance_1.0_lite")] Seedance10Lite, + #[serde(rename = "sora_2")] + Sora2, + #[serde(rename = "veo_2")] Veo2, @@ -55,7 +60,7 @@ pub struct EnqueueImageToVideoRequest { /// The model to use. pub model: Option, - /// Currently REQUIRED. + /// Currently REQUIRED by some downstream models. /// Image media file; the starting frame of the video. /// TODO: In the future we may support base64 images, URLs, or file paths here. pub image_media_token: Option, @@ -105,6 +110,7 @@ pub async fn enqueue_image_to_video_command( fal_task_queue: State<'_, FalTaskQueue>, storyteller_creds_manager: State<'_, StorytellerCredentialManager>, sora_task_queue: State<'_, SoraTaskQueue>, + sora_creds_manager: State<'_, SoraCredentialManager>, ) -> Response { info!("enqueue_image_to_video_command called"); @@ -117,6 +123,7 @@ pub async fn enqueue_image_to_video_command( &provider_priority_store, &task_database, &fal_creds_manager, + &sora_creds_manager, &storyteller_creds_manager, &fal_task_queue, ).await; @@ -189,20 +196,32 @@ pub async fn handle_request( provider_priority_store: &ProviderPriorityStore, task_database: &TaskDatabase, fal_creds_manager: &FalCredentialManager, + sora_creds_manager: &SoraCredentialManager, storyteller_creds_manager: &StorytellerCredentialManager, fal_task_queue: &FalTaskQueue, ) -> Result { - let result = handle_video( - request, - &app, - &app_env_configs, - &app_data_root, - &provider_priority_store, - &fal_creds_manager, - &storyteller_creds_manager, - &fal_task_queue, - ).await; + let result = match request.model { + Some(VideoModel::Sora2) => { + handle_sora2_video( + &request, + provider_priority_store, + sora_creds_manager, + ).await + } + _ => { + handle_video( + request, + &app, + &app_env_configs, + &app_data_root, + &provider_priority_store, + &fal_creds_manager, + &storyteller_creds_manager, + &fal_task_queue, + ).await + } + }; let success_event = match result { Err(err) => return Err(err), diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs index 4664bcdb98..3e290361d8 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs @@ -8,6 +8,7 @@ use crate::core::events::generation_events::generation_enqueue_success_event::Ge use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProAspectRatio, GenerateKling16ProImageToVideoRequest}; use artcraft_api_defs::generate::video::generate_kling_2_1_master_image_to_video::{GenerateKling21MasterAspectRatio, GenerateKling21MasterImageToVideoRequest}; use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProAspectRatio, GenerateKling21ProImageToVideoRequest}; @@ -73,6 +74,11 @@ pub async fn handle_video_artcraft( None => { return Err(GenerateError::no_model_specified()); } + Some(VideoModel::Sora2) => { + return Err(GenerateError::AnyhowError( + anyhow!("wrong logic: another branch should handle this: {:?}", + request.model))); + } Some(VideoModel::Kling16Pro) => { info!("enqueue Kling 1.6 Pro with Artcraft API"); selected_model = Some(GenerationModel::Kling1_6); diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/mod.rs index 5fab18c141..48074cb5ee 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/mod.rs @@ -1,2 +1,3 @@ pub mod enqueue_image_to_video_command; -mod generic; \ No newline at end of file +mod generic; +pub mod sora2; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video.rs new file mode 100644 index 0000000000..d9d558d02d --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video.rs @@ -0,0 +1,49 @@ +use crate::core::commands::enqueue::generate_error::GenerateError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; +use crate::core::commands::enqueue::image_inpaint::enqueue_image_inpaint_command::EnqueueInpaintImageCommand; +use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::EnqueueImageToVideoRequest; +use crate::core::commands::enqueue::image_to_video::sora2::handle_sora2_video_sora::handle_sora2_video_sora; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use log::info; +use tauri::AppHandle; + +pub async fn handle_sora2_video( + request: &EnqueueImageToVideoRequest, + provider_priority_store: &ProviderPriorityStore, + sora_creds_manager: &SoraCredentialManager, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + info!("Providers by priority: {:?}", priority); + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough (for now) + } + Provider::Artcraft => { + // Fallthrough + } + Provider::Sora => { + info!("Dispatching Sora2 via OpenAI Sora..."); + return handle_sora2_video_sora( + request, + sora_creds_manager, + ).await; + } + } + } + + Err(GenerateError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video_sora.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video_sora.rs new file mode 100644 index 0000000000..9494f11046 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/handle_sora2_video_sora.rs @@ -0,0 +1,73 @@ +use crate::core::commands::enqueue::generate_error::GenerateError; +use crate::core::commands::enqueue::image_to_video::enqueue_image_to_video_command::EnqueueImageToVideoRequest; +use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::events::generation_events::common::GenerationModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use enums::common::generation_provider::GenerationProvider; +use enums::tauri::tasks::task_type::TaskType; +use log::{error, info}; +use openai_sora_client::recipes::generate_sora2_video::generate_sora2_video_with_session_auto_renew::generate_sora2_video_with_session_auto_renew; +use openai_sora_client::recipes::list_sora2_drafts::list_sora2_drafts_with_session_auto_renew::list_sora2_drafts_with_session_auto_renew; +use openai_sora_client::requests::generate_sora2_video::generate_sora2_video::GenerateSora2VideoArgs; +use tauri::AppHandle; +use tokens::tokens::media_files::MediaFileToken; + +pub (super) async fn handle_sora2_video_sora( + request: &EnqueueImageToVideoRequest, + sora_creds_manager: &SoraCredentialManager, +) -> Result { + + let creds = match sora_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(GenerateError::needs_sora_credentials()); + }, + }; + + // TODO: Handle input images. + // let image_media_token = match &request.image_media_token { + // Some(token) => token.clone(), + // None => { + // return Err(GenerateError::required_source_image_not_provided()); + // }, + // }; + + info!("Calling Sora 2 generate..."); + + let result = generate_sora2_video_with_session_auto_renew( + GenerateSora2VideoArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + credentials: &creds, + request_timeout: None, + } + ).await; + + let job_id = match result { + Ok((response, maybe_new_session)) => { + info!("Successfully enqueued Sora2 Video: {}", response.task_id); + + if let Some(new_creds) = maybe_new_session { + if let Err(err) = sora_creds_manager.set_credentials(&new_creds) { + error!("Failed to save renewed Sora credentials: {:?}", err); + } + } + + response.task_id + } + Err(err) => { + error!("Failed to use Sora2 video: {:?}", err); + return Err(GenerateError::from(err)); + } + }; + + Ok(TaskEnqueueSuccess { + provider: GenerationProvider::Sora, + model: Some(GenerationModel::Sora2), + provider_job_id: Some(job_id.to_string()), + task_type: TaskType::VideoGeneration, + }) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/mod.rs new file mode 100644 index 0000000000..d259ca7083 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/sora2/mod.rs @@ -0,0 +1,2 @@ +pub (super) mod handle_sora2_video; +pub (super) mod handle_sora2_video_sora; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs b/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs index efc126f7fd..c49e71a2cd 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/task_enqueue_success.rs @@ -70,6 +70,7 @@ impl TaskEnqueueSuccess{ Some(GenerationModel::Kling21Pro) => Some(TaskModelType::Kling21Pro), Some(GenerationModel::Kling21Master) => Some(TaskModelType::Kling21Master), Some(GenerationModel::Seedance10Lite) => Some(TaskModelType::Seedance10Lite), + Some(GenerationModel::Sora2) => Some(TaskModelType::Sora2), Some(GenerationModel::Veo2) => Some(TaskModelType::Veo2), Some(GenerationModel::Veo3) => Some(TaskModelType::Veo3), Some(GenerationModel::Veo3Fast) => Some(TaskModelType::Veo3Fast), diff --git a/crates/desktop/artcraft/src/core/events/generation_events/common.rs b/crates/desktop/artcraft/src/core/events/generation_events/common.rs index 80f84c2e53..3d45ce600e 100644 --- a/crates/desktop/artcraft/src/core/events/generation_events/common.rs +++ b/crates/desktop/artcraft/src/core/events/generation_events/common.rs @@ -55,6 +55,8 @@ pub enum GenerationModel { Recraft3, #[serde(rename = "sora")] Sora, + #[serde(rename = "sora_2")] + Sora2, #[serde(rename = "seedance_1.0_lite")] Seedance10Lite, #[serde(rename = "veo_2")] diff --git a/crates/desktop/artcraft/src/services/sora/commands/check_sora_session_command.rs b/crates/desktop/artcraft/src/services/sora/commands/check_sora_session_command.rs index 9bd00e1dc9..8bb241e972 100644 --- a/crates/desktop/artcraft/src/services/sora/commands/check_sora_session_command.rs +++ b/crates/desktop/artcraft/src/services/sora/commands/check_sora_session_command.rs @@ -3,7 +3,7 @@ use chrono::{TimeDelta, Utc}; use errors::AnyhowResult; use log::{error, info}; use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; -use openai_sora_client::requests::list_tasks::list_classic_tasks::list_tasks; +use openai_sora_client::requests::list_classic_tasks::list_classic_tasks::list_classic_tasks; use serde_derive::Serialize; use std::ops::Add; use tauri::{AppHandle, State}; @@ -88,7 +88,7 @@ async fn do_check( info!("Attempting to upgrade session..."); upgraded = maybe_upgrade_or_renew_session(&mut creds).await?; info!("Attempting to check session..."); - list_tasks(&creds).await?; // Intent here is to make sure the upgrade request worked. + list_classic_tasks(&creds).await?; // Intent here is to make sure the upgrade request worked. sora_creds_manager.record_credential_success()?; } diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_failed_generations.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_failed_generations.rs index 8c3b6b5a87..b1d7738ba9 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_failed_generations.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_failed_generations.rs @@ -9,7 +9,7 @@ use enums::tauri::tasks::task_status; use errors::AnyhowResult; use log::info; use openai_sora_client::requests::common::task_id::TaskId; -use openai_sora_client::requests::list_tasks::list_classic_tasks::PartialTaskResponse; +use openai_sora_client::requests::list_classic_tasks::list_classic_tasks::PartialTaskResponse; use sqlite_tasks::queries::list_tasks_by_provider_and_status::Task; use sqlite_tasks::queries::update_task_status::{update_task_status, UpdateTaskArgs}; use std::collections::HashMap; diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_successful_generations.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_successful_generations.rs index e573249a3a..6943047454 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_successful_generations.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/handle_successful_generations.rs @@ -22,7 +22,7 @@ use once_cell::sync::Lazy; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; use openai_sora_client::recipes::list_classic_sora_tasks_with_session_auto_renew::list_classic_sora_tasks_with_session_auto_renew; use openai_sora_client::requests::common::task_id::TaskId; -use openai_sora_client::requests::list_tasks::list_classic_tasks::{PartialGeneration, PartialTaskResponse}; +use openai_sora_client::requests::list_classic_tasks::list_classic_tasks::{PartialGeneration, PartialTaskResponse}; use reqwest::Url; use sqlite_tasks::queries::list_tasks_by_provider_and_status::{list_tasks_by_provider_and_status, ListTasksByProviderAndStatusArgs, Task, TaskList}; use sqlite_tasks::queries::update_task_status::{update_task_status, UpdateTaskArgs}; diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs index 391513826a..600b9f1f1f 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs @@ -24,7 +24,7 @@ use once_cell::sync::Lazy; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; use openai_sora_client::recipes::list_classic_sora_tasks_with_session_auto_renew::list_classic_sora_tasks_with_session_auto_renew; use openai_sora_client::requests::common::task_id::TaskId; -use openai_sora_client::requests::list_tasks::list_classic_tasks::TaskStatus; +use openai_sora_client::requests::list_classic_tasks::list_classic_tasks::TaskStatus; use reqwest::Url; use sqlite_tasks::queries::list_tasks_by_provider_and_status::{list_tasks_by_provider_and_status, ListTasksByProviderAndStatusArgs, Task, TaskList}; use sqlite_tasks::queries::update_task_status::{update_task_status, UpdateTaskArgs}; @@ -78,7 +78,7 @@ async fn local_task_polling_loop( provider: GenerationProvider::Sora, task_statuses: &TASK_DATABASE_PENDING_STATUSES, }).await?; - + if local_sqlite_tasks.tasks.is_empty() { // No need to poll if we don't have pending tasks. tokio::time::sleep(std::time::Duration::from_millis(2_000)).await; @@ -128,7 +128,7 @@ async fn poll_sora_tasks( }) .collect::>(); - let (sora_response, maybe_new_creds) = + let (sora_response, maybe_new_creds) = list_classic_sora_tasks_with_session_auto_renew(&sora_creds).await?; if let Some(new_creds) = maybe_new_creds { diff --git a/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs b/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs index 6e0f8abdcb..7d64855feb 100644 --- a/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs +++ b/crates/schema/public/enums/src/tauri/tasks/task_model_type.rs @@ -46,6 +46,8 @@ pub enum TaskModelType { Kling21Master, #[serde(rename = "seedance_1.0_lite")] Seedance10Lite, + #[serde(rename = "sora_2")] + Sora2, #[serde(rename = "veo_2")] Veo2, #[serde(rename = "veo_3")] @@ -86,6 +88,7 @@ impl TaskModelType { Self::Kling21Pro => "kling_2.1_pro", Self::Kling21Master => "kling_2.1_master", Self::Seedance10Lite => "seedance_1.0_lite", + Self::Sora2 => "sora_2", Self::Veo2 => "veo_2", Self::Veo3 => "veo_3", Self::Veo3Fast => "veo_3_fast", @@ -114,6 +117,7 @@ impl TaskModelType { "kling_2.1_pro" => Ok(Self::Kling21Pro), "kling_2.1_master" => Ok(Self::Kling21Master), "seedance_1.0_lite" => Ok(Self::Seedance10Lite), + "sora_2" => Ok(Self::Sora2), "veo_2" => Ok(Self::Veo2), "veo_3" => Ok(Self::Veo3), "veo_3_fast" => Ok(Self::Veo3Fast), @@ -145,6 +149,7 @@ impl TaskModelType { Self::Kling21Pro, Self::Kling21Master, Self::Seedance10Lite, + Self::Sora2, Self::Veo2, Self::Veo3, Self::Veo3Fast, @@ -182,6 +187,7 @@ mod tests { assert_serialization(TaskModelType::Kling21Pro, "kling_2.1_pro"); assert_serialization(TaskModelType::Kling21Master, "kling_2.1_master"); assert_serialization(TaskModelType::Seedance10Lite, "seedance_1.0_lite"); + assert_serialization(TaskModelType::Sora2, "sora_2"); assert_serialization(TaskModelType::Veo2, "veo_2"); assert_serialization(TaskModelType::Veo3, "veo_3"); assert_serialization(TaskModelType::Veo3Fast, "veo_3_fast"); @@ -209,6 +215,7 @@ mod tests { assert_eq!(TaskModelType::Kling21Pro.to_str(), "kling_2.1_pro"); assert_eq!(TaskModelType::Kling21Master.to_str(), "kling_2.1_master"); assert_eq!(TaskModelType::Seedance10Lite.to_str(), "seedance_1.0_lite"); + assert_eq!(TaskModelType::Sora2.to_str(), "sora_2"); assert_eq!(TaskModelType::Veo2.to_str(), "veo_2"); assert_eq!(TaskModelType::Veo3.to_str(), "veo_3"); assert_eq!(TaskModelType::Veo3Fast.to_str(), "veo_3_fast"); @@ -236,6 +243,7 @@ mod tests { assert_eq!(TaskModelType::from_str("kling_2.1_pro").unwrap(), TaskModelType::Kling21Pro); assert_eq!(TaskModelType::from_str("kling_2.1_master").unwrap(), TaskModelType::Kling21Master); assert_eq!(TaskModelType::from_str("seedance_1.0_lite").unwrap(), TaskModelType::Seedance10Lite); + assert_eq!(TaskModelType::from_str("sora_2").unwrap(), TaskModelType::Sora2); assert_eq!(TaskModelType::from_str("veo_2").unwrap(), TaskModelType::Veo2); assert_eq!(TaskModelType::from_str("veo_3").unwrap(), TaskModelType::Veo3); assert_eq!(TaskModelType::from_str("veo_3_fast").unwrap(), TaskModelType::Veo3Fast); @@ -247,7 +255,7 @@ mod tests { #[test] fn all_variants() { let mut variants = TaskModelType::all_variants(); - assert_eq!(variants.len(), 20); + assert_eq!(variants.len(), 21); // Image models assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(TaskModelType::Flux1Schnell)); @@ -265,6 +273,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(TaskModelType::Kling21Pro)); assert_eq!(variants.pop_first(), Some(TaskModelType::Kling21Master)); assert_eq!(variants.pop_first(), Some(TaskModelType::Seedance10Lite)); + assert_eq!(variants.pop_first(), Some(TaskModelType::Sora2)); assert_eq!(variants.pop_first(), Some(TaskModelType::Veo2)); assert_eq!(variants.pop_first(), Some(TaskModelType::Veo3)); assert_eq!(variants.pop_first(), Some(TaskModelType::Veo3Fast));