diff --git a/crates/_types/tokens/src/macros.rs b/crates/_types/tokens/src/macros.rs index 3a0bcffc53..99cca525fa 100644 --- a/crates/_types/tokens/src/macros.rs +++ b/crates/_types/tokens/src/macros.rs @@ -56,6 +56,13 @@ macro_rules! impl_crockford_generator { $t(token) } + #[inline] + pub fn entropy_suffix(&self) -> &str { + &self.0 + .strip_prefix($variant.prefix()) + .unwrap_or(&self.0) + } + #[inline] pub fn entropic_character_len() -> usize { let token_prefix = Self::token_prefix(); diff --git a/crates/_types/tokens/tests/media_upload_token_tests.rs b/crates/_types/tokens/tests/media_upload_token_tests.rs index 1ad8323a0b..26c98c4fe5 100644 --- a/crates/_types/tokens/tests/media_upload_token_tests.rs +++ b/crates/_types/tokens/tests/media_upload_token_tests.rs @@ -163,4 +163,16 @@ mod crockford_traits { let token_string = MediaUploadToken::generate().to_string(); assert!(token_string.starts_with(prefix)); } + + #[test] + fn entropy_suffix() { + let token = MediaUploadToken::new_from_str("mu_foo"); + assert_eq!(token.entropy_suffix(), "foo"); + + let token = MediaUploadToken::new_from_str("bar"); + assert_eq!(token.entropy_suffix(), "bar"); + + let token = MediaUploadToken::generate(); + assert_eq!(token.entropy_suffix().len(), 29); + } } diff --git a/crates/_types/tokens/tests/user_token_tests.rs b/crates/_types/tokens/tests/user_token_tests.rs index c14b3edc6e..6013cdd8e9 100644 --- a/crates/_types/tokens/tests/user_token_tests.rs +++ b/crates/_types/tokens/tests/user_token_tests.rs @@ -163,4 +163,16 @@ mod crockford_traits { let token_string = UserToken::generate().to_string(); assert!(token_string.starts_with(prefix)); } + + #[test] + fn entropy_suffix() { + let token = UserToken::new_from_str("U:foo"); + assert_eq!(token.entropy_suffix(), "foo"); + + let token = UserToken::new_from_str("bar"); + assert_eq!(token.entropy_suffix(), "bar"); + + let token = UserToken::generate(); + assert_eq!(token.entropy_suffix().len(), 13); + } } diff --git a/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs b/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs index 44729ac5f0..447ade17d7 100644 --- a/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs +++ b/crates/programs/aichatbot-sidecar/src/persistence/save_directory.rs @@ -3,6 +3,7 @@ use log::info; use path_absolutize::Absolutize; use errors::{anyhow, AnyhowResult}; use hashing::sha256::sha256_hash_string::sha256_hash_string; +use tokens::tokens::news_stories::NewsStoryToken; use crate::persistence::hashed_directory_path::hashed_directory_path; #[derive(Clone)] @@ -18,6 +19,7 @@ impl SaveDirectory { } } + // TODO: Move web pages for news under "news_stories/{hashed_url_path}/" pub fn directory_for_webpage_url(&self, url: &str) -> AnyhowResult { let url_hash = sha256_hash_string(url)?; let directory_path= hashed_directory_path(&url_hash); @@ -30,6 +32,20 @@ impl SaveDirectory { Ok(directory) } + // TODO: Move web pages for news under "news_stories/{hashed_url_path}/audio/" + // This requires associating the job with the URL. + pub fn directory_for_audio(&self, news_story_token: &NewsStoryToken) -> AnyhowResult { + let entropy_part = news_story_token.entropy_suffix(); + let directory_path= hashed_directory_path(&entropy_part); + + let directory = self.directory.clone() + .join("audio/") + .join(directory_path) + .join(news_story_token.as_str()); + + Ok(directory) + } + pub fn html_file_for_webpage_url(&self, url: &str) -> AnyhowResult { Ok(self.directory_for_webpage_url(url)?.join("webpage.html")) } @@ -46,6 +62,11 @@ impl SaveDirectory { Ok(self.directory_for_webpage_url(url)?.join("speakable_monologue.yaml")) } + pub fn audio_wav_file_for_news_story(&self, news_story_token: &NewsStoryToken, sequence_order: i64) -> AnyhowResult { + let filename = format!("{}.wav", sequence_order); + Ok(self.directory_for_audio(news_story_token)?.join(filename)) + } + /// This is just the first directory structure, which is sequential audio files. /// We'll be using a database and well-formed filesystem layout later. pub fn get_audio_files_dir_v1(&self) -> PathBuf { diff --git a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs index ed7204363f..32e993e106 100644 --- a/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs +++ b/crates/programs/aichatbot-sidecar/src/workers/audio/fakeyou_audio_download/process_single_record.rs @@ -1,3 +1,4 @@ +use std::io::Cursor; use crate::shared_state::job_state::JobState; use errors::{anyhow, AnyhowResult}; use fakeyou_client::api::tts_inference::{CreateTtsInferenceRequest, TtsInferenceJobStatusStatePayload}; @@ -7,6 +8,7 @@ use sqlite_queries::queries::by_table::tts_render_targets::list::tts_render_targ use sqlite_queries::queries::by_table::tts_render_targets::update::update_tts_render_target_successfully_downloaded::Args; use sqlite_queries::queries::by_table::tts_render_targets::update::update_tts_render_target_successfully_downloaded::update_tts_render_target_successfully_downloaded; use std::sync::Arc; +use tokens::tokens::news_stories::NewsStoryToken; use tokens::tokens::tts_models::TtsModelToken; use tokens::tokens::tts_render_tasks::TtsRenderTaskToken; @@ -46,7 +48,7 @@ pub async fn process_single_record(target: &TtsRenderTarget, job_state: &Arc { - process_download(&target.token, &payload, job_state).await?; + process_download(&target, &payload, job_state).await?; } } }, @@ -55,7 +57,7 @@ pub async fn process_single_record(target: &TtsRenderTarget, job_state: &Arc) -> AnyhowResult<()> { +async fn process_download(target: &TtsRenderTarget, payload: &TtsInferenceJobStatusStatePayload, job_state: &Arc) -> AnyhowResult<()> { let tts_result_token = match &payload.maybe_result_token { None => return Ok(()), // TODO: Wait Some(result_token) => result_token.to_string(), @@ -67,9 +69,25 @@ async fn process_download(task_token: &TtsRenderTaskToken, payload: &TtsInferenc }; let audio_url = get_audio_url(&bucket_path, true); + let news_story_token = NewsStoryToken::new_from_str(&target.story_token); + + { + let directory = job_state.save_directory.directory_for_audio(&news_story_token)?; + std::fs::create_dir_all(&directory)?; + } + + { + let download_filename = job_state.save_directory + .audio_wav_file_for_news_story(&news_story_token, target.sequence_order)?; + + let response = reqwest::get(&audio_url).await?; + let mut file = std::fs::File::create(download_filename)?; + let mut content = Cursor::new(response.bytes().await?); + std::io::copy(&mut content, &mut file)?; + } update_tts_render_target_successfully_downloaded(Args { - tts_render_task_token: task_token, + tts_render_task_token: &target.token, tts_result_token: &tts_result_token, result_url: &audio_url, sqlite_pool: &job_state.sqlite_pool,