From 08e85604994060b9929e2cff812683db155ea26c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 21 Apr 2023 23:27:20 -0700 Subject: [PATCH] voice conversion result upload (WIP) --- crates/_types/buckets/src/public/mod.rs | 3 +- .../voice_conversion_results/directory.rs | 63 +++++++++++ .../public/voice_conversion_results/mod.rs | 2 + .../voice_conversion_results/original_file.rs | 105 ++++++++++++++++++ .../job_types/vc/so_vits_svc/process_job.rs | 12 +- 5 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 crates/_types/buckets/src/public/voice_conversion_results/directory.rs create mode 100644 crates/_types/buckets/src/public/voice_conversion_results/mod.rs create mode 100644 crates/_types/buckets/src/public/voice_conversion_results/original_file.rs diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index 38f2952846..f27eafae3d 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -1,2 +1,3 @@ pub mod media_uploads; -pub mod public_path; \ No newline at end of file +pub mod public_path; +pub mod voice_conversion_results; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/voice_conversion_results/directory.rs b/crates/_types/buckets/src/public/voice_conversion_results/directory.rs new file mode 100644 index 0000000000..30b0b009e8 --- /dev/null +++ b/crates/_types/buckets/src/public/voice_conversion_results/directory.rs @@ -0,0 +1,63 @@ +use crate::public::public_path::PublicPath; +use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + +// TODO: Generate these from a macro. + +// TODO: Use a central path registry for quick reference +const DIRECTORY: &'static str = "/voice_conversion"; + +/// Directory for voice conversion results. +/// Each result gets its own directory so that we can store the result +/// alongside metadata and re-processed derivatives. +#[derive(Clone)] +pub struct VoiceConversionResultDirectory { + object_hash: String, + directory: String, +} + +impl PublicPath for VoiceConversionResultDirectory {} + +impl VoiceConversionResultDirectory { + + pub fn from_public_bucket_directory_hash(object_hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let middle = hashed_directory_path_long_string(object_hash); + let directory = format!("{}/{}{}", DIRECTORY, middle, object_hash); + Self { + object_hash: object_hash.to_string(), + directory, + } + } + + pub fn get_directory_path_str(&self) -> &str { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.object_hash + } +} + +#[cfg(test)] +mod tests { + use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory; + + #[test] + pub fn get_directory_path_str() { + let directory = VoiceConversionResultDirectory::from_public_bucket_directory_hash("abcdefghijk"); + assert_eq!(directory.get_directory_path_str(), "/voice_conversion/a/b/c/d/e/abcdefghijk"); + } + + #[test] + pub fn get_directory_path_str_short_name() { + let directory = VoiceConversionResultDirectory::from_public_bucket_directory_hash("foo"); + assert_eq!(directory.get_directory_path_str(), "/voice_conversion/f/o/foo"); + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let directory = VoiceConversionResultDirectory::from_public_bucket_directory_hash(hash); + assert_eq!(directory.get_object_hash(), hash); + } +} diff --git a/crates/_types/buckets/src/public/voice_conversion_results/mod.rs b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs new file mode 100644 index 0000000000..42ede393cf --- /dev/null +++ b/crates/_types/buckets/src/public/voice_conversion_results/mod.rs @@ -0,0 +1,2 @@ +pub mod directory; +pub mod original_file; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs b/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs new file mode 100644 index 0000000000..f1a650265e --- /dev/null +++ b/crates/_types/buckets/src/public/voice_conversion_results/original_file.rs @@ -0,0 +1,105 @@ +use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::public_path::PublicPath; +use crockford::crockford_entropy_lower; +use std::path::PathBuf; + +const ORIGINAL_FILE_PREFIX : &'static str = "fakeyou_"; +const ORIGINAL_FILE_SUFFIX : &'static str = ".wav"; + +// TODO: Generate these from a macro. + +/// The original user upload file. +/// It may have derivative files (down samples, crops, etc.) that live alongside it. +#[derive(Clone)] +pub struct VoiceConversionResultOriginalFilePath { + directory: MediaUploadDirectory, + basename: String, + full_object_path: String, +} + +impl PublicPath for VoiceConversionResultOriginalFilePath {} + +impl VoiceConversionResultOriginalFilePath { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let directory = MediaUploadDirectory::from_object_hash(hash); + let basename = format!("{}{}{}", ORIGINAL_FILE_PREFIX, hash, ORIGINAL_FILE_SUFFIX); + let full_object_path = format!("{}/{}{}{}", directory.get_directory_path_str(), ORIGINAL_FILE_PREFIX, hash, ORIGINAL_FILE_SUFFIX); + Self { + directory, + basename, + full_object_path, + } + } + + pub fn get_full_object_path_str(&self) -> &str { + &self.full_object_path + } + + pub fn to_full_object_pathbuf(&self) -> PathBuf { + PathBuf::from(&self.full_object_path) + } + + pub fn get_directory(&self) -> &MediaUploadDirectory { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.directory.get_object_hash() + } + + pub fn get_basename(&self) -> &str { + &self.basename + } +} + +#[cfg(test)] +mod tests { + use crate::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; + use std::path::PathBuf; + + #[test] + pub fn generate_new_entropy() { + let file = VoiceConversionResultOriginalFilePath::generate_new(); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_directory().get_object_hash().len(), 32); + } + + #[test] + pub fn get_full_object_path_str() { + let file = VoiceConversionResultOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk/fakeyou_abcdefghijk.wav"); + } + + #[test] + pub fn to_full_object_pathbuf() { + let file = VoiceConversionResultOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/media_upload/a/b/c/d/e/abcdefghijk/fakeyou_abcdefghijk.wav")); + } + + #[test] + pub fn get_full_object_path_str_short_name() { + let file = VoiceConversionResultOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/media_upload/f/o/foo/fakeyou_foo.wav"); + } + + #[test] + pub fn get_full_object_path_str_starts_with_directory() { + let file = VoiceConversionResultOriginalFilePath::from_object_hash("abcdefghijk"); + let full_path = file.get_full_object_path_str(); + assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let file = VoiceConversionResultOriginalFilePath::from_object_hash(hash); + assert_eq!(file.get_object_hash(), hash); + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index 9b12f9331f..64331c5ed6 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -24,6 +24,7 @@ use std::path::{Path, PathBuf}; use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; use tempdir::TempDir; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; use tokens::files::media_upload::MediaUploadToken; use tts_common::clean_symbols::clean_symbols; @@ -192,15 +193,16 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result) -> Result