diff --git a/Cargo.lock b/Cargo.lock index b9d40aa954..149424ab0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2628,6 +2628,7 @@ dependencies = [ name = "fakeyou_client" version = "0.0.1" dependencies = [ + "backoff", "bytes 1.3.0", "chrono", "enums", diff --git a/crates/lib/fakeyou_client/Cargo.toml b/crates/lib/fakeyou_client/Cargo.toml index 9f79f68198..2d33b2a31d 100644 --- a/crates/lib/fakeyou_client/Cargo.toml +++ b/crates/lib/fakeyou_client/Cargo.toml @@ -31,6 +31,7 @@ serde_json = "1.0.64" toml = "0.5.8" # HTTP +backoff = { version = "0.4.0", features = ["futures"] } # Exponential backoff retry reqwest = { version = "0.11", features = ["json", "cookies"] } # HTTP Client [dev-dependencies] diff --git a/crates/lib/fakeyou_client/src/fakeyou_api_client.rs b/crates/lib/fakeyou_client/src/fakeyou_api_client.rs index 7344e99fa3..9dc8c9ed8a 100644 --- a/crates/lib/fakeyou_client/src/fakeyou_api_client.rs +++ b/crates/lib/fakeyou_client/src/fakeyou_api_client.rs @@ -1,9 +1,12 @@ -use std::sync::Arc; +use backoff::future::retry; +use backoff::{ExponentialBackoff, ExponentialBackoffBuilder}; use crate::api::tts_inference::{CreateTtsInferenceRequest, CreateTtsInferenceResponse, TtsInferenceJobStatus}; -use errors::AnyhowResult; -use reqwest::{Client, ClientBuilder, RequestBuilder, Url}; -use reqwest::cookie::Jar; use crate::credentials::FakeYouCredentials; +use errors::{anyhow, AnyhowResult}; +use reqwest::cookie::Jar; +use reqwest::{Client, ClientBuilder, RequestBuilder, Url}; +use std::sync::Arc; +use std::time::Duration; const AUTHORIZATION_HEADER: &'static str = "Authorization"; const SESSION_COOKIE_NAME : &'static str = "session"; @@ -55,18 +58,37 @@ impl FakeYouApiClient { // TODO: need to yield better, more "library"-appropriate errors. pub async fn create_tts_inference(&self, request: CreateTtsInferenceRequest<'_>) -> AnyhowResult { - let url = format!("https://{}/tts/inference", self.api_domain); + let response = self.do_create_tts_inference(&request).await?; + let response = serde_json::from_str(&response)?; + Ok(response) + } + pub async fn create_tts_inference_with_backoff(&self, request: CreateTtsInferenceRequest<'_>) -> AnyhowResult { + //let backoff = ExponentialBackoff::default(); + let mut backoff = ExponentialBackoffBuilder::new() + .with_initial_interval(Duration::from_secs(3)) // Initial retry interval: 3 seconds + .with_randomization_factor(0.5) // 50% below, 50% above interval + .with_multiplier(2.0) // Double the wait + .with_max_elapsed_time(Some(Duration::from_secs(60))) + //.with_max_interval(Duration::from_secs(30)) + .build(); + let response = retry(backoff, || async { + Ok(self.do_create_tts_inference(&request).await?) + }).await?; + + let response = serde_json::from_str(&response)?; + Ok(response) + } + + pub async fn do_create_tts_inference(&self, request: &CreateTtsInferenceRequest<'_>) -> Result { + let url = format!("https://{}/tts/inference", self.api_domain); let response = self.add_credentials(self.client .post(url)) - .json(&request) + .json(request) .send() .await? .text() .await?; - - let response = serde_json::from_str(&response)?; - Ok(response) } diff --git a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_create/process_single_record.rs b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_create/process_single_record.rs index 6ee5dd39ba..0eb28aca28 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_create/process_single_record.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_create/process_single_record.rs @@ -21,7 +21,7 @@ pub async fn process_single_record(target: &TtsRenderTask, job_state: &Arc { if !res.success { - error!("unknown problem in submitting to FakeYou"); + error!("unknown problem in submitting TTS request to FakeYou"); update_tts_render_task_unsuccessfully_submitted(UnsuccessfulArgs { tts_render_task_token: &target.token, diff --git a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs index 7a9a67827e..f573ac505d 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs @@ -40,7 +40,7 @@ pub async fn process_single_record(target: &TtsRenderTask, job_state: &Arc { match res.state { None => { - error!("unknown problem in submitting to FakeYou"); + error!("unknown problem in checking FakeYou TTS inference job status"); // TODO: Errors //update_tts_render_task_unsuccessfully_submitted(UnsuccessfulArgs { // tts_render_task_token: &target.token,