Merge pull request #113 from storytold/echelon-voice-conversion-media-files

voice conversion --> media files
This commit is contained in:
Brandon Thomas authored and GitHub committed 2023-10-20 06:14:19 -04:00
commit fbed7cd530
11 files changed
+458 -117

No files matched your search

@@ -21,6 +21,11 @@ pub enum IdCategory {
#[serde(rename = "lipsync_animation")]
LipsyncAnimation,
/// Results from voice conversion (which may live in the media_files table)
/// Applies for RVC and SVC
#[serde(rename = "voice_conversion")]
VoiceConversion,
/// Zs dataset which lives in the zs_voice_datasets table
#[serde(rename = "zs_dataset")]
ZsDataset,
@@ -40,6 +45,7 @@ impl IdCategory {
match self {
Self::MediaFile => "media_file",
Self::LipsyncAnimation => "lipsync_animation",
Self::VoiceConversion => "voice_conversion",
Self::ZsDataset => "zs_dataset",
Self::ZsVoice => "zs_voice",
}
@@ -49,6 +55,7 @@ impl IdCategory {
match value {
"media_file" => Ok(Self::MediaFile),
"lipsync_animation" => Ok(Self::LipsyncAnimation),
"voice_conversion" => Ok(Self::VoiceConversion),
"zs_dataset" => Ok(Self::ZsDataset),
"zs_voice" => Ok(Self::ZsVoice),
_ => Err(format!("invalid value: {:?}", value)),
@@ -61,6 +68,7 @@ impl IdCategory {
BTreeSet::from([
Self::MediaFile,
Self::LipsyncAnimation,
Self::VoiceConversion,
Self::ZsDataset,
Self::ZsVoice,
])
@@ -72,41 +80,70 @@ mod tests {
use crate::by_table::generic_synthetic_ids::id_category::IdCategory;
use crate::test_helpers::assert_serialization;
#[test]
fn test_serialization() {
assert_serialization(IdCategory::MediaFile, "media_file");
assert_serialization(IdCategory::LipsyncAnimation, "lipsync_animation");
mod explicit_checks {
use super::*;
#[test]
fn test_serialization() {
assert_serialization(IdCategory::MediaFile, "media_file");
assert_serialization(IdCategory::LipsyncAnimation, "lipsync_animation");
assert_serialization(IdCategory::VoiceConversion, "voice_conversion");
assert_serialization(IdCategory::ZsDataset, "zs_dataset");
assert_serialization(IdCategory::ZsVoice, "zs_voice");
}
#[test]
fn to_str() {
assert_eq!(IdCategory::MediaFile.to_str(), "media_file");
assert_eq!(IdCategory::LipsyncAnimation.to_str(), "lipsync_animation");
assert_eq!(IdCategory::VoiceConversion.to_str(), "voice_conversion");
assert_eq!(IdCategory::ZsDataset.to_str(), "zs_dataset");
assert_eq!(IdCategory::ZsVoice.to_str(), "zs_voice");
}
#[test]
fn from_str() {
assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile);
assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimation);
assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversion);
assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZsDataset);
assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZsVoice);
}
#[test]
fn all_variants() {
// Static check
let mut variants = IdCategory::all_variants();
assert_eq!(variants.len(), 5);
assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile));
assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimation));
assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversion));
assert_eq!(variants.pop_first(), Some(IdCategory::ZsDataset));
assert_eq!(variants.pop_first(), Some(IdCategory::ZsVoice));
assert_eq!(variants.pop_first(), None);
// Generated check
use strum::IntoEnumIterator;
assert_eq!(IdCategory::all_variants().len(), IdCategory::iter().len());
}
}
#[test]
fn to_str() {
assert_eq!(IdCategory::MediaFile.to_str(), "media_file");
assert_eq!(IdCategory::LipsyncAnimation.to_str(), "lipsync_animation");
assert_eq!(IdCategory::ZsDataset.to_str(), "zs_dataset");
assert_eq!(IdCategory::ZsVoice.to_str(), "zs_voice");
}
mod mechanical_checks {
use super::*;
#[test]
fn from_str() {
assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile);
assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimation);
assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZsDataset);
assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZsVoice);
}
#[test]
fn variant_length() {
use strum::IntoEnumIterator;
assert_eq!(IdCategory::all_variants().len(), IdCategory::iter().len());
}
#[test]
fn all_variants() {
// Static check
let mut variants = IdCategory::all_variants();
assert_eq!(variants.len(), 4);
assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile));
assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimation));
assert_eq!(variants.pop_first(), Some(IdCategory::ZsDataset));
assert_eq!(variants.pop_first(), Some(IdCategory::ZsVoice));
assert_eq!(variants.pop_first(), None);
// Generated check
use strum::IntoEnumIterator;
assert_eq!(IdCategory::all_variants().len(), IdCategory::iter().len());
#[test]
fn round_trip() {
for variant in IdCategory::all_variants() {
assert_eq!(variant, IdCategory::from_str(variant.to_str()).unwrap());
assert_eq!(variant, IdCategory::from_str(&format!("{}", variant)).unwrap());
assert_eq!(variant, IdCategory::from_str(&format!("{:?}", variant)).unwrap());
}
}
}
}
@@ -1,3 +1,5 @@
use std::collections::BTreeSet;
use serde::Deserialize;
use serde::Serialize;
#[cfg(test)]
@@ -9,11 +11,19 @@ use strum::EnumIter;
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MediaFileOriginModelType {
/// RVC (v2) voice conversion models
#[serde(rename = "rvc_v2")]
RvcV2,
/// SadTalker -- v1, we may add another enum value for future versions
SadTalker,
/// so-vits-svc voice conversion models
#[serde(rename = "so_vits_svc")]
SoVitsSvc,
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
@@ -24,16 +34,30 @@ impl_mysql_enum_coders!(MediaFileOriginModelType);
impl MediaFileOriginModelType {
pub fn to_str(&self) -> &'static str {
match self {
Self::RvcV2 => "rvc_v2",
Self::SadTalker => "sad_talker",
Self::SoVitsSvc => "so_vits_svc",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"rvc_v2" => Ok(Self::RvcV2),
"sad_talker" => Ok(Self::SadTalker),
"so_vits_svc" => Ok(Self::SoVitsSvc),
_ => Err(format!("invalid value: {:?}", value)),
}
}
pub fn all_variants() -> BTreeSet<Self> {
// NB: BTreeSet is sorted
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
Self::RvcV2,
Self::SadTalker,
Self::SoVitsSvc,
])
}
}
#[cfg(test)]
@@ -41,19 +65,59 @@ mod tests {
use crate::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use crate::test_helpers::assert_serialization;
#[test]
fn test_serialization() {
assert_serialization(MediaFileOriginModelType::SadTalker, "sad_talker");
mod explicit_checks {
use super::*;
#[test]
fn test_serialization() {
assert_serialization(MediaFileOriginModelType::RvcV2, "rvc_v2");
assert_serialization(MediaFileOriginModelType::SadTalker, "sad_talker");
assert_serialization(MediaFileOriginModelType::SoVitsSvc, "so_vits_svc");
}
#[test]
fn test_to_str() {
assert_eq!(MediaFileOriginModelType::RvcV2.to_str(), "rvc_v2");
assert_eq!(MediaFileOriginModelType::SadTalker.to_str(), "sad_talker");
assert_eq!(MediaFileOriginModelType::SoVitsSvc.to_str(), "so_vits_svc");
}
#[test]
fn test_from_str() {
assert_eq!(MediaFileOriginModelType::from_str("rvc_v2").unwrap(), MediaFileOriginModelType::RvcV2);
assert_eq!(MediaFileOriginModelType::from_str("sad_talker").unwrap(), MediaFileOriginModelType::SadTalker);
assert_eq!(MediaFileOriginModelType::from_str("so_vits_svc").unwrap(), MediaFileOriginModelType::SoVitsSvc);
assert!(MediaFileOriginModelType::from_str("foo").is_err());
}
#[test]
fn all_variants() {
let mut variants = MediaFileOriginModelType::all_variants();
assert_eq!(variants.len(), 3);
assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2));
assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker));
assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc));
assert_eq!(variants.pop_first(), None);
}
}
#[test]
fn test_to_str() {
assert_eq!(MediaFileOriginModelType::SadTalker.to_str(), "sad_talker");
}
#[test]
fn test_from_str() {
assert_eq!(MediaFileOriginModelType::from_str("sad_talker").unwrap(), MediaFileOriginModelType::SadTalker);
assert!(MediaFileOriginModelType::from_str("foo").is_err());
mod mechanical_checks {
use super::*;
#[test]
fn variant_length() {
use strum::IntoEnumIterator;
assert_eq!(MediaFileOriginModelType::all_variants().len(), MediaFileOriginModelType::iter().len());
}
#[test]
fn round_trip() {
for variant in MediaFileOriginModelType::all_variants() {
assert_eq!(variant, MediaFileOriginModelType::from_str(variant.to_str()).unwrap());
assert_eq!(variant, MediaFileOriginModelType::from_str(&format!("{}", variant)).unwrap());
assert_eq!(variant, MediaFileOriginModelType::from_str(&format!("{:?}", variant)).unwrap());
}
}
}
}
@@ -27,6 +27,10 @@ pub enum MediaFileOriginProductCategory {
#[serde(rename = "unknown")]
Unknown,
/// Voice conversion (either RVC or SVC)
#[serde(rename = "voice_conversion")]
VoiceConversion,
/// Media files created by (or uploaded for) the Zero Shot voice product.
#[serde(rename = "zs_voice")]
ZeroShotVoice,
@@ -42,6 +46,7 @@ impl MediaFileOriginProductCategory {
match self {
Self::FaceAnimator => "face_animator",
Self::Unknown => "unknown",
Self::VoiceConversion => "voice_conversion",
Self::ZeroShotVoice => "zs_voice",
}
}
@@ -50,6 +55,7 @@ impl MediaFileOriginProductCategory {
match value {
"face_animator" => Ok(Self::FaceAnimator),
"unknown" => Ok(Self::Unknown),
"voice_conversion" => Ok(Self::VoiceConversion),
"zs_voice" => Ok(Self::ZeroShotVoice),
_ => Err(format!("invalid value: {:?}", value)),
}
@@ -61,6 +67,7 @@ impl MediaFileOriginProductCategory {
BTreeSet::from([
Self::FaceAnimator,
Self::Unknown,
Self::VoiceConversion,
Self::ZeroShotVoice,
])
}
@@ -78,6 +85,7 @@ mod tests {
fn test_serialization() {
assert_serialization(MediaFileOriginProductCategory::FaceAnimator, "face_animator");
assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown");
assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion");
assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice");
}
@@ -85,6 +93,7 @@ mod tests {
fn to_str() {
assert_eq!(MediaFileOriginProductCategory::FaceAnimator.to_str(), "face_animator");
assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown");
assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion");
assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice");
}
@@ -92,15 +101,17 @@ mod tests {
fn from_str() {
assert_eq!(MediaFileOriginProductCategory::from_str("face_animator").unwrap(), MediaFileOriginProductCategory::FaceAnimator);
assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown);
assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion);
assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice);
}
#[test]
fn all_variants() {
let mut variants = MediaFileOriginProductCategory::all_variants();
assert_eq!(variants.len(), 3);
assert_eq!(variants.len(), 4);
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice));
assert_eq!(variants.pop_first(), None);
}
@@ -10,6 +10,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use crate::helpers::boolean_converters::i8_to_bool;
#[derive(Debug)]
pub struct GenericInferenceJobStatus {
pub job_token: InferenceJobToken,
@@ -34,6 +35,7 @@ pub struct GenericInferenceJobStatus {
/// Details about the user's original inference request
/// (We may want to present it in the "pending" UI.)
#[derive(Debug)]
pub struct RequestDetails {
pub inference_category: InferenceCategory,
pub maybe_model_type: Option<String>, // TODO: Strongly type
@@ -45,6 +47,7 @@ pub struct RequestDetails {
}
/// Details about the generated result
#[derive(Debug)]
pub struct ResultDetails {
pub entity_type: String,
pub entity_token: String,
@@ -94,6 +97,7 @@ SELECT
tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,
voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,
media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,
media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,
media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,
@@ -144,12 +148,22 @@ WHERE jobs.token = ?
// NB: A bit of a hack. We store TTS results with a full path.
// Going forward, all other record types will store a hash.
let (bucket_path_is_hash, maybe_public_bucket_hash) = match record.inference_category {
let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category {
InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
};
// NB: We've moved voice conversion out of their own table and into media_files
// This check should allow for graceful migration to the new end-state.
match record.maybe_result_entity_type.as_deref() {
Some("media_file") => {
bucket_path_is_hash = true;
maybe_public_bucket_hash = record.maybe_media_file_public_bucket_directory_hash.as_deref();
}
_ => {}
}
let maybe_result_details = record
.maybe_result_entity_type
.as_deref()
@@ -0,0 +1,173 @@
use anyhow::anyhow;
use sqlx;
use sqlx::MySqlPool;
use enums::by_table::generic_synthetic_ids::id_category::IdCategory;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use errors::AnyhowResult;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::users::UserToken;
use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id;
pub enum VoiceConversionModelType {
SoVitsSvc,
RvcV2,
}
pub struct InsertMediaFileArgs<'a> {
pub pool: &'a MySqlPool,
pub job: &'a AvailableInferenceJob,
pub voice_conversion_type: VoiceConversionModelType,
pub maybe_mime_type: Option<&'a str>,
pub file_size_bytes: u64,
pub duration_millis: u64,
pub sha256_checksum: &'a str,
// TODO: Media duration.
//pub duration_millis: u64,
pub public_bucket_directory_hash: &'a str,
pub maybe_public_bucket_prefix: Option<&'a str>,
pub maybe_public_bucket_extension: Option<&'a str>,
pub is_on_prem: bool,
pub worker_hostname: &'a str,
pub worker_cluster: &'a str,
}
pub async fn insert_media_file_from_voice_conversion(
args: InsertMediaFileArgs<'_>
) -> AnyhowResult<(MediaFileToken, u64)>
{
let result_token = MediaFileToken::generate();
let origin_model_type = match args.voice_conversion_type {
VoiceConversionModelType::SoVitsSvc => MediaFileOriginModelType::SoVitsSvc,
VoiceConversionModelType::RvcV2 => MediaFileOriginModelType::RvcV2,
};
let mut maybe_creator_file_synthetic_id : Option<u64> = None;
let mut maybe_creator_category_synthetic_id : Option<u64> = None;
let mut transaction = args.pool.begin().await?;
if let Some(creator_user_token) = args.job.maybe_creator_user_token.as_deref() {
let user_token = UserToken::new_from_str(creator_user_token);
let next_media_file_id = transactional_increment_generic_synthetic_id(
&user_token,
IdCategory::MediaFile,
&mut transaction
).await?;
let next_voice_conversion_id = transactional_increment_generic_synthetic_id(
&user_token,
IdCategory::VoiceConversion,
&mut transaction
).await?;
maybe_creator_file_synthetic_id = Some(next_media_file_id);
maybe_creator_category_synthetic_id = Some(next_voice_conversion_id);
}
let vc_model_token = args.job.maybe_model_token.as_deref();
let creator_ip_address = args.job.creator_ip_address.as_str();
let creator_set_visibility = args.job.creator_set_visibility.clone();
const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference;
const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::VoiceConversion;
const MEDIA_TYPE : MediaFileType = MediaFileType::Audio;
let record_id = {
let query_result = sqlx::query!(
r#"
INSERT INTO media_files
SET
token = ?,
origin_category = ?,
origin_product_category = ?,
maybe_origin_model_type = ?,
maybe_origin_model_token = ?,
media_type = ?,
maybe_mime_type = ?,
file_size_bytes = ?,
checksum_sha2 = ?,
public_bucket_directory_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
maybe_creator_user_token = ?,
creator_ip_address = ?,
creator_set_visibility = ?,
maybe_creator_file_synthetic_id = ?,
maybe_creator_category_synthetic_id = ?,
is_generated_on_prem = ?,
generated_by_worker = ?,
generated_by_cluster = ?
"#,
result_token.as_str(),
ORIGIN_CATEGORY.to_str(),
ORIGIN_PRODUCT_CATEGORY.to_str(),
origin_model_type.to_str(),
args.job.maybe_model_token,
MEDIA_TYPE.to_str(),
args.maybe_mime_type,
args.file_size_bytes,
args.sha256_checksum,
args.public_bucket_directory_hash,
args.maybe_public_bucket_prefix,
args.maybe_public_bucket_extension,
args.job.maybe_creator_user_token,
args.job.creator_ip_address,
args.job.creator_set_visibility.to_str(),
maybe_creator_file_synthetic_id,
maybe_creator_category_synthetic_id,
args.is_on_prem,
args.worker_hostname,
args.worker_cluster
)
.execute(&mut transaction)
.await;
let record_id = match query_result {
Ok(res) => {
res.last_insert_id()
},
Err(err) => {
// TODO: handle better
//transaction.rollback().await?;
return Err(anyhow!("Mysql error: {:?}", err));
}
};
record_id
};
transaction.commit().await?;
Ok((result_token, record_id))
}
@@ -1,3 +1,4 @@
pub mod get_media_file;
pub mod insert_media_file_from_cli_tool;
pub mod insert_media_file_from_face_animation;
pub mod insert_media_file_from_cli_tool;
pub mod insert_media_file_from_voice_conversion;
@@ -43,13 +43,11 @@ SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
# #
# ================================= #
SO_VITS_SVC_MODEL_CHECK_ROOT_DIRECTORY="/models/voice_conversion/so-vits-svc"
SO_VITS_SVC_MODEL_CHECK_COMMAND="python3 -m so_vits_svc_fork.fakeyou_infer"
SO_VITS_SVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE="8acacf30bb2e"
SO_VITS_SVC_MODEL_CHECK_ROOT_DIRECTORY="/models/voice_conversion/so-vits-svc/src"
SO_VITS_SVC_MODEL_CHECK_COMMAND="/install/python/bin/python3.10 fakeyou_infer.py" # NB: so-vits-svc-paul-2
SO_VITS_SVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE="1136fe8b50d5"
SO_VITS_SVC_MODEL_CHECK_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vits-svc/example_config.json"
SO_VITS_SVC_MODEL_CHECK_MAYBE_DEFAULT_TEST_WAV_PATH="/models/voice_conversion/so-vits-svc/example.wav"
#SO_VITS_SVC_MODEL_CHECK_COMMAND="/common/so-vits-svc-code/install/python/bin/python3.10 -m so_vits_svc_fork.fakeyou_infer"
#SO_VITS_SVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
# ================================= #
# #
@@ -60,10 +60,9 @@ FFMPEG_LOGO_LOGO_PATH="%storyteller_rust%/includes/container_includes/image_asse
HUBERT_PATH="/tmp/hubert/checkpoint_best_legacy_500.pt"
SO_VITS_SVC_INFERENCE_ROOT_DIRECTORY="/models/voice_conversion/so-vits-svc/src" #NB: so-vits-svc-paul-2
SO_VITS_SVC_INFERENCE_MAYBE_DOCKER_IMAGE="247952d65c82" # so-vits-svc-paul-2: --f0-method
SO_VITS_SVC_INFERENCE_MAYBE_DOCKER_IMAGE="1136fe8b50d5" # so-vits-svc-paul-2: --f0-method
SO_VITS_SVC_INFERENCE_COMMAND="/install/python/bin/python3.10 fakeyou_infer.py" # NB: so-vits-svc-paul-2
SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vits-svc/example_config.json"
#SO_VITS_SVC_INFERENCE_MAYBE_VENV_COMMAND="source python/bin/activate"
# ================================= #
# #
@@ -3,6 +3,7 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath;
@@ -12,13 +13,14 @@ use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
use filesys::file_size::file_size;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use media::decode_basic_audio_info::decode_basic_audio_file_info;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference;
use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs};
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::users::UserToken;
@@ -28,6 +30,9 @@ use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::InferenceArgs;
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
const BUCKET_FILE_PREFIX : &str = "fakeyou_rvc_";
const BUCKET_FILE_EXTENSION : &str = ".wav";
pub struct RvcV2ProcessJobArgs<'a> {
pub job_dependencies: &'a JobDependencies,
pub job: &'a AvailableInferenceJob,
@@ -267,8 +272,6 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
info!("Checking that output files exist...");
check_file_exists(&output_audio_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
//check_file_exists(&output_metadata_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
//check_file_exists(&output_spectrogram_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
info!("Interrogating result file properties...");
@@ -286,11 +289,18 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
warn!("Required a full decode of the output data to get duration! That's inefficient!");
}
info!("Calculating sha256...");
let file_checksum = sha256_hash_file(&output_audio_fs_path)
.map_err(|err| {
ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err))
})?;
// TODO: Make a new python image that generates spectrograms from any audio file.
let file_metadata = FileMetadata {
duration_millis: audio_info.duration_millis,
mimetype: maybe_mimetype,
mimetype: maybe_mimetype.clone(),
file_size_bytes,
};
@@ -301,7 +311,15 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
job_progress_reporter.log_status("uploading result")
.map_err(|e| ProcessSingleJobError::Other(e))?;
let result_bucket_location = VoiceConversionResultOriginalFilePath::generate_new();
// let result_bucket_location = VoiceConversionResultOriginalFilePath::generate_new();
//
// let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
//
// info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf);
let result_bucket_location = MediaFileBucketPath::generate_new(
Some(BUCKET_FILE_PREFIX),
Some(BUCKET_FILE_EXTENSION));
let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
@@ -318,24 +336,6 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
safe_delete_temp_file(&output_audio_fs_path);
// // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== //
//
// let spectrogram_result_object_path = args.job_dependencies.bucket_path_unifier.tts_inference_spectrogram_output_path(
// &job.uuid_idempotency_token); // TODO: Don't use this!
//
// info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path);
//
// info!("Uploading spectrogram...");
//
// args.job_dependencies.public_bucket_client.upload_filename_with_content_type(
// &spectrogram_result_object_path,
// &output_spectrogram_fs_path,
// "application/json")
// .await
// .map_err(|e| ProcessSingleJobError::Other(e))?;
//
// safe_delete_temp_file(&output_spectrogram_fs_path);
// ==================== DELETE DOWNLOADED FILE ==================== //
// NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk...
@@ -345,16 +345,34 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
info!("Saving vc inference record...");
let (inference_result_token, id) = insert_voice_conversion_result(InsertArgs {
// let (inference_result_token, id) = insert_voice_conversion_result(InsertArgs {
// pool: &args.job_dependencies.mysql_pool,
// job: &job,
// public_bucket_hash: result_bucket_location.get_object_hash(),
// file_size_bytes: file_metadata.file_size_bytes,
// duration_millis: file_metadata.duration_millis.unwrap_or(0),
// is_on_prem: args.job_dependencies.container.is_on_prem,
// worker_hostname: &args.job_dependencies.container.hostname,
// worker_cluster: &args.job_dependencies.container.cluster_name,
// is_debug_worker: args.job_dependencies.worker_details.is_debug_worker
// })
// .await
// .map_err(|e| ProcessSingleJobError::Other(e))?;
let (inference_result_token, id) = insert_media_file_from_voice_conversion(InsertMediaFileArgs {
pool: &args.job_dependencies.mysql_pool,
job: &job,
public_bucket_hash: result_bucket_location.get_object_hash(),
voice_conversion_type: VoiceConversionModelType::SoVitsSvc,
maybe_mime_type: maybe_mimetype.as_deref(),
file_size_bytes: file_metadata.file_size_bytes,
duration_millis: file_metadata.duration_millis.unwrap_or(0),
sha256_checksum: &file_checksum,
public_bucket_directory_hash: result_bucket_location.get_object_hash(),
maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX),
maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION),
is_on_prem: args.job_dependencies.container.is_on_prem,
worker_hostname: &args.job_dependencies.container.hostname,
worker_cluster: &args.job_dependencies.container.cluster_name,
is_debug_worker: args.job_dependencies.worker_details.is_debug_worker
})
.await
.map_err(|e| ProcessSingleJobError::Other(e))?;
@@ -2,6 +2,7 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath;
@@ -11,13 +12,14 @@ use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
use filesys::file_size::file_size;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use media::decode_basic_audio_info::decode_basic_audio_file_info;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference;
use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs};
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::users::UserToken;
@@ -27,6 +29,9 @@ use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::{Devi
use crate::job_dependencies::JobDependencies;
use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket;
const BUCKET_FILE_PREFIX : &str = "fakeyou_svc_";
const BUCKET_FILE_EXTENSION : &str = ".wav";
pub struct SoVitsSvcProcessJobArgs<'a> {
pub job_dependencies: &'a JobDependencies,
pub job: &'a AvailableInferenceJob,
@@ -131,14 +136,10 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
let input_wav_path = original_media_upload_fs_path;
let output_audio_fs_path = work_temp_dir.path().join("output.wav");
//let output_metadata_fs_path = temp_dir.path().join("metadata.json");
//let output_spectrogram_fs_path = temp_dir.path().join("spectrogram.json");
info!("Running VC inference...");
info!("Expected output audio filename: {:?}", &output_audio_fs_path);
//info!("Expected output metadata filename: {:?}", &output_metadata_fs_path);
//info!("Expected output spectrogram filename: {:?}", &output_spectrogram_fs_path);
// TODO: Limit output length for premium.
@@ -200,8 +201,6 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
info!("Checking that output files exist...");
check_file_exists(&output_audio_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
//check_file_exists(&output_metadata_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
//check_file_exists(&output_spectrogram_fs_path).map_err(|e| ProcessSingleJobError::Other(e))?;
info!("Interrogating result file properties...");
@@ -219,28 +218,41 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
warn!("Required a full decode of the output data to get duration! That's inefficient!");
}
info!("Calculating sha256...");
let file_checksum = sha256_hash_file(&output_audio_fs_path)
.map_err(|err| {
ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err))
})?;
// TODO: Make a new python image that generates spectrograms from any audio file.
let file_metadata = FileMetadata {
duration_millis: audio_info.duration_millis,
mimetype: maybe_mimetype,
mimetype: maybe_mimetype.clone(),
file_size_bytes,
};
//safe_delete_temp_file(&output_metadata_fs_path);
// ==================== UPLOAD AUDIO TO BUCKET ==================== //
job_progress_reporter.log_status("uploading result")
.map_err(|e| ProcessSingleJobError::Other(e))?;
let result_bucket_location = VoiceConversionResultOriginalFilePath::generate_new();
// let result_bucket_location = VoiceConversionResultOriginalFilePath::generate_new();
//
// let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
//
// info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf);
let result_bucket_location = MediaFileBucketPath::generate_new(
Some(BUCKET_FILE_PREFIX),
Some(BUCKET_FILE_EXTENSION));
let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf();
info!("Audio destination bucket path: {:?}", &result_bucket_object_pathbuf);
info!("Uploading audio...");
info!("Uploading audio media file...");
args.job_dependencies.public_bucket_client.upload_filename_with_content_type(
&result_bucket_object_pathbuf,
@@ -251,24 +263,6 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
safe_delete_temp_file(&output_audio_fs_path);
// // ==================== UPLOAD SPECTROGRAM TO BUCKETS ==================== //
//
// let spectrogram_result_object_path = args.job_dependencies.bucket_path_unifier.tts_inference_spectrogram_output_path(
// &job.uuid_idempotency_token); // TODO: Don't use this!
//
// info!("Spectrogram destination bucket path: {:?}", &spectrogram_result_object_path);
//
// info!("Uploading spectrogram...");
//
// args.job_dependencies.public_bucket_client.upload_filename_with_content_type(
// &spectrogram_result_object_path,
// &output_spectrogram_fs_path,
// "application/json")
// .await
// .map_err(|e| ProcessSingleJobError::Other(e))?;
//
// safe_delete_temp_file(&output_spectrogram_fs_path);
// ==================== DELETE DOWNLOADED FILE ==================== //
// NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk...
@@ -278,16 +272,34 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
info!("Saving vc inference record...");
let (inference_result_token, id) = insert_voice_conversion_result(InsertArgs {
// let (inference_result_token, id) = insert_voice_conversion_result(InsertArgs {
// pool: &args.job_dependencies.mysql_pool,
// job: &job,
// public_bucket_hash: result_bucket_location.get_object_hash(),
// file_size_bytes: file_metadata.file_size_bytes,
// duration_millis: file_metadata.duration_millis.unwrap_or(0),
// is_on_prem: args.job_dependencies.container.is_on_prem,
// worker_hostname: &args.job_dependencies.container.hostname,
// worker_cluster: &args.job_dependencies.container.cluster_name,
// is_debug_worker: args.job_dependencies.worker_details.is_debug_worker
// })
// .await
// .map_err(|e| ProcessSingleJobError::Other(e))?;
let (inference_result_token, id) = insert_media_file_from_voice_conversion(InsertMediaFileArgs {
pool: &args.job_dependencies.mysql_pool,
job: &job,
public_bucket_hash: result_bucket_location.get_object_hash(),
voice_conversion_type: VoiceConversionModelType::SoVitsSvc,
maybe_mime_type: maybe_mimetype.as_deref(),
file_size_bytes: file_metadata.file_size_bytes,
duration_millis: file_metadata.duration_millis.unwrap_or(0),
sha256_checksum: &file_checksum,
public_bucket_directory_hash: result_bucket_location.get_object_hash(),
maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX),
maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION),
is_on_prem: args.job_dependencies.container.is_on_prem,
worker_hostname: &args.job_dependencies.container.hostname,
worker_cluster: &args.job_dependencies.container.cluster_name,
is_debug_worker: args.job_dependencies.worker_details.is_debug_worker
})
.await
.map_err(|e| ProcessSingleJobError::Other(e))?;
@@ -316,7 +328,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
Ok(JobSuccessResult {
maybe_result_entity: Some(ResultEntity {
entity_type: InferenceResultType::VoiceConversion,
entity_type: InferenceResultType::MediaFile,
entity_token: inference_result_token.to_string(),
}),
inference_duration,
@@ -233,9 +233,23 @@ pub async fn get_inference_job_status_handler(
result_details.public_bucket_location_or_hash
}
InferenceCategory::VoiceConversion => {
VoiceConversionResultOriginalFilePath::from_object_hash(&result_details.public_bucket_location_or_hash)
.get_full_object_path_str()
.to_string()
match result_details.entity_type.as_str() {
"media_file" => {
// NB: We're migrating voice conversion to media_files.
MediaFileBucketPath::from_object_hash(
&result_details.public_bucket_location_or_hash,
result_details.maybe_media_file_public_bucket_prefix.as_deref(),
result_details.maybe_media_file_public_bucket_extension.as_deref())
.get_full_object_path_str()
.to_string()
}
_ => {
// NB: This is the old voice conversion result pathing.
VoiceConversionResultOriginalFilePath::from_object_hash(&result_details.public_bucket_location_or_hash)
.get_full_object_path_str()
.to_string()
}
}
}
};