diff --git a/_sql/migrations/2023-10-13-093804_alter_zs_voice_dataset_samples_add_idempotency/down.sql b/_sql/migrations/2023-10-13-093804_alter_zs_voice_dataset_samples_add_idempotency/down.sql new file mode 100644 index 0000000000..eb2f675ffe --- /dev/null +++ b/_sql/migrations/2023-10-13-093804_alter_zs_voice_dataset_samples_add_idempotency/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE zs_voice_dataset_samples DROP COLUMN uuid_idempotency_token; diff --git a/_sql/migrations/2023-10-13-093804_alter_zs_voice_dataset_samples_add_idempotency/up.sql b/_sql/migrations/2023-10-13-093804_alter_zs_voice_dataset_samples_add_idempotency/up.sql new file mode 100644 index 0000000000..dfb85eeb01 --- /dev/null +++ b/_sql/migrations/2023-10-13-093804_alter_zs_voice_dataset_samples_add_idempotency/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE zs_voice_dataset_samples + ADD COLUMN uuid_idempotency_token VARCHAR(36) NOT NULL + AFTER token; + +ALTER TABLE zs_voice_dataset_samples + ADD UNIQUE INDEX unique_index_uuid_idempotency_token (uuid_idempotency_token); diff --git a/_sql/migrations_squashed/media_files.sql b/_sql/migrations_squashed/media_files.sql index 8728d7a463..90fc13febb 100644 --- a/_sql/migrations_squashed/media_files.sql +++ b/_sql/migrations_squashed/media_files.sql @@ -22,7 +22,8 @@ CREATE TABLE media_files ( -- Broad category for where the file came from: -- * 'inference' for inference output -- * 'processed' for processed file (eg. mp3 encoding, stem splitting, etc.) - -- * 'upload' for direct user upload + -- * 'upload' for direct user upload (from the filesystem) + -- * 'device_api' for direct user uploads recorded using Browser/Device APIs. origin_category VARCHAR(16) NOT NULL, -- TODO: Remove default value once records backfilled and queries updated. diff --git a/_sql/migrations_squashed/zs_voices/zs_voice_dataset_samples.sql b/_sql/migrations_squashed/zs_voices/zs_voice_dataset_samples.sql new file mode 100644 index 0000000000..db0c98f260 --- /dev/null +++ b/_sql/migrations_squashed/zs_voices/zs_voice_dataset_samples.sql @@ -0,0 +1,89 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- These are the individual files in each dataset. +-- Files can be added and removed from a dataset at any time. +CREATE TABLE zs_voice_dataset_samples ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" for joins, API, etc. (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- Idempotency token from client + -- This is so the frontend client doesn't submit duplicate uploads. + uuid_idempotency_token VARCHAR(36) NOT NULL, + + -- ========== ZERO SHOT FOREIGN KEYS AND STATE========== + + -- The zero shot dataset that was used to create this voice. + -- The dataset will only be accessible to the original author of the voice. + -- The original author can create multiple voices from the same dataset. + dataset_token VARCHAR(32) NOT NULL, + + -- Foreign key to the uploaded media file. + -- There was an option of handling uploads and file management separately, but by electing to store the dataset + -- samples in the "media_files" table, they can be reused by our other code, queries, machinery, etc. + -- Note that this requires that the "media_files" record must be an audio (not video) file (which our code will + -- enforce), and that it is given a category of "voice_zero_shot" so it isn't co-mingled with other uploads/outputs + media_file_token VARCHAR(32) NOT NULL, + + -- Some data sets require raw transcripts for the audio. + -- If required, they'll live here. + maybe_raw_transcription TEXT DEFAULT NULL, + + -- A way to order the data set samples + -- We don't necessarily need to reimplement file reordering. + -- order_id INT DEFAULT NULL, + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes last. + -- Fine details of moderator actions are kept in a separate `audit_logs` table. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + UNIQUE KEY (uuid_idempotency_token), + + KEY fk_dataset_token (dataset_token), + KEY fk_media_file_token (media_file_token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at (user_deleted_at), + KEY index_mod_deleted_at (mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/zs_voices/zs_voice_datasets.sql b/_sql/migrations_squashed/zs_voices/zs_voice_datasets.sql new file mode 100644 index 0000000000..e06af2203a --- /dev/null +++ b/_sql/migrations_squashed/zs_voices/zs_voice_datasets.sql @@ -0,0 +1,109 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- This table describes a zero shot voice in the process of being created (or later reused / edited / enhanced) +-- These are not usable voices until their vector embeddings have been created, and this table does not serve +-- the production voice list. +-- +-- Multiple voices can be created from a single "dataset" as the data set is mutable over time. Users can add or +-- remove new samples to the data set at any time. +CREATE TABLE zs_voice_datasets ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" for joins, API, etc. (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- NB: No idempotency token. + -- Store it in another table to prevent duplicate record creation from rapid user inputs. + + -- ========== VOICE DETAILS ========== + + -- The "title" of the voice. This can be the name and additional information as supplied by the user. + -- When the voice embeddings are created, this will be copied to the produced "voice" (embeddings) record. + title VARCHAR(255) NOT NULL, + + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela" + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en") + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + + -- TODO: misc speaker metadata + -- We'll want rich metadata for search + -- When the voice embeddings are created, this will be copied to the produced "voice" (embeddings) record. + -- We don't want a hard link between the dataset and the produced voice as the voice is static once + -- created (with the exception of regenerating embeddings), but the dataset my drift as the user experiments, + -- adds/removes samples, etc. + -- series_name VARCHAR(255) DEFAULT NULL, + -- speaker_name VARCHAR(255) DEFAULT NULL, + -- speaker_gender VARCHAR(255) DEFAULT NULL, + -- speaker_age VARCHAR(255) DEFAULT NULL, + -- speaker_voice_style VARCHAR(255) DEFAULT NULL, + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- The creator can set a desired visibility for their data. + -- This does not always apply to every upload type. + -- Additionally, some upload types may require moderator approval prior + -- to being publicly listed, and this field has no bearing on that. + -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes last. + -- Fine details of moderator actions are kept in a separate `audit_logs` table. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + + KEY index_ietf_primary_language_subtag (ietf_primary_language_subtag), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at (user_deleted_at), + KEY index_mod_deleted_at (mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/zs_voices/zs_voices.sql b/_sql/migrations_squashed/zs_voices/zs_voices.sql new file mode 100644 index 0000000000..353dec935f --- /dev/null +++ b/_sql/migrations_squashed/zs_voices/zs_voices.sql @@ -0,0 +1,157 @@ +-- NB: This is a manually squashed view of all the CREATE and ALTER statements, +-- with comments attached to the fields for centralized documentation. + +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- This table contains both zero shot TTS voices (speaker embeddings) and zero shot V2V voices (speaker embeddings). +-- These are the zero shot voices that are usable by our users. +CREATE TABLE zs_voices ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" for joins, API, etc. (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- NB: No idempotency token. + -- Store it in another table to prevent duplicate record creation from rapid user inputs. + + -- ========== ZERO SHOT FOREIGN KEYS AND STATE========== + + -- The zero shot dataset that was used to create this voice. + -- The dataset will only be accessible to the original author of the voice. + -- The original author can create multiple voices from the same dataset. + dataset_token VARCHAR(32) NOT NULL, + + -- When datasets are edited, we bump their version. + -- This tells us which version of the dataset created this voice (not that we should preserve + -- record of which samples are in which version.) + -- We can surface this to the users as well. + dataset_version INT NOT NULL DEFAULT 0, + + -- ========== MODEL DETAILS ========== + + -- The class or category of model/algorithm this voice is intended for. + -- * 'tts' for zero shot TTS + -- * 'vc' for zero shot voice conversion + -- * 'pitch' for pitch correction models (hypothetical) + -- * (Since this isn't an enum, the list of values can be expanded.) + model_category VARCHAR(16) NOT NULL, + + -- The model used for the voice: + -- * 'vall-e-x' for Vall-E-X + -- * 'ns2vc' for NaturalSpeech2 voice conversion (hypothetical) + -- * (Since this isn't an enum, the list of values can be expanded.) + model_type VARCHAR(16) NOT NULL, + + -- The version of the model. + -- If we deploy new model weights for the model, we may need to regenerate + -- all of the corresponding speaker embeddings. This lets us know which voices + -- are the old version. + -- In some cases, we may need to preserve, hide, or roll back to older versions. + model_version INTEGER UNSIGNED NOT NULL DEFAULT 0, + + -- What kind of features these are: spectrogram, encodec, etc. + -- Technically we shouldn't need this, as the 2-tuple (model_type, model_version) implicitly knows + -- what its features should be, but this is probably useful for our own analysis and may be easier + -- to deal with. + -- + -- Example values: + -- * 'spectrogram' + -- * 'encodec' + -- * (Since this isn't an enum, the list of values can be expanded.) + encoding_type VARCHAR(16) NOT NULL, + + -- ========== VOICE DETAILS ========== + + -- The "title" of the voice. This can be the name and additional information as supplied by the user. + title VARCHAR(255) NOT NULL, + + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela" + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en") + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + + -- TODO: misc speaker metadata + -- We'll want rich metadata for search + -- series_name VARCHAR(255) DEFAULT NULL, + -- speaker_name VARCHAR(255) DEFAULT NULL, + -- speaker_gender VARCHAR(255) DEFAULT NULL, + -- speaker_age VARCHAR(255) DEFAULT NULL, + -- speaker_voice_style VARCHAR(255) DEFAULT NULL, + + -- ========== STORAGE DETAILS ========== + + -- The hash for the bucket directory that contains the original upload + -- eg. /speaker_vectors/{bucket_hash}/{voice_model_version}.bin + bucket_hash VARCHAR(32) NOT NULL, + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- The creator can set a desired visibility for their data. + -- This does not always apply to every upload type. + -- Additionally, some upload types may require moderator approval prior + -- to being publicly listed, and this field has no bearing on that. + -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes last. + -- Fine details of moderator actions are kept in a separate `audit_logs` table. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + + -- TODO: These are not correct. Fix these before landing / running migrations. + KEY index_model_category (model_category), + KEY index_model_type (model_type), + KEY index_model_type_and_version (model_type, model_version), + KEY index_encoding_type (encoding_type), + KEY index_ietf_primary_language_subtag (ietf_primary_language_subtag), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at (user_deleted_at), + KEY index_mod_deleted_at (mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs index 5eac9a9a9f..5db6daa07e 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_category.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use serde::Deserialize; use serde::Serialize; #[cfg(test)] @@ -9,8 +10,8 @@ use strum::EnumIter; /// /// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] pub enum MediaFileOriginCategory { /// ML model inference output - uploaded models or zero shot. Inference, @@ -18,8 +19,11 @@ pub enum MediaFileOriginCategory { /// Processed results - (we don't have these systems yet, but eg. trim, transcode, etc). Processed, - /// User uploaded files + /// User uploaded files (from their filesystem) Upload, + + /// User uploaded files recorded directly from their device (browser, mobile), typically using device APIs. + DeviceApi, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -33,6 +37,7 @@ impl MediaFileOriginCategory { Self::Inference => "inference", Self::Processed => "processed", Self::Upload => "upload", + Self::DeviceApi => "device_api", } } @@ -41,9 +46,21 @@ impl MediaFileOriginCategory { "inference" => Ok(Self::Inference), "processed" => Ok(Self::Processed), "upload" => Ok(Self::Upload), + "device_api" => Ok(Self::DeviceApi), _ => Err(format!("invalid value: {:?}", value)), } } + + pub fn all_variants() -> BTreeSet { + // NB: BTreeSet is sorted + // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. + BTreeSet::from([ + Self::Inference, + Self::Processed, + Self::Upload, + Self::DeviceApi, + ]) + } } #[cfg(test)] @@ -51,25 +68,62 @@ mod tests { use crate::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use crate::test_helpers::assert_serialization; - #[test] - fn test_serialization() { - assert_serialization(MediaFileOriginCategory::Inference, "inference"); - assert_serialization(MediaFileOriginCategory::Processed, "processed"); - assert_serialization(MediaFileOriginCategory::Upload, "upload"); + mod explicit_checks { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(MediaFileOriginCategory::Inference, "inference"); + assert_serialization(MediaFileOriginCategory::Processed, "processed"); + assert_serialization(MediaFileOriginCategory::Upload, "upload"); + assert_serialization(MediaFileOriginCategory::DeviceApi, "device_api"); + } + + #[test] + fn test_to_str() { + assert_eq!(MediaFileOriginCategory::Inference.to_str(), "inference"); + assert_eq!(MediaFileOriginCategory::Processed.to_str(), "processed"); + assert_eq!(MediaFileOriginCategory::Upload.to_str(), "upload"); + assert_eq!(MediaFileOriginCategory::DeviceApi.to_str(), "device_api"); + } + + #[test] + fn test_from_str() { + assert_eq!(MediaFileOriginCategory::from_str("inference").unwrap(), MediaFileOriginCategory::Inference); + assert_eq!(MediaFileOriginCategory::from_str("processed").unwrap(), MediaFileOriginCategory::Processed); + assert_eq!(MediaFileOriginCategory::from_str("upload").unwrap(), MediaFileOriginCategory::Upload); + assert_eq!(MediaFileOriginCategory::from_str("device_api").unwrap(), MediaFileOriginCategory::DeviceApi); + assert!(MediaFileOriginCategory::from_str("foo").is_err()); + } + + #[test] + fn all_variants() { + let mut variants = MediaFileOriginCategory::all_variants(); + assert_eq!(variants.len(), 4); + assert_eq!(variants.pop_first(), Some(MediaFileOriginCategory::Inference)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginCategory::Processed)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginCategory::Upload)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginCategory::DeviceApi)); + assert_eq!(variants.pop_first(), None); + } } - #[test] - fn test_to_str() { - assert_eq!(MediaFileOriginCategory::Inference.to_str(), "inference"); - assert_eq!(MediaFileOriginCategory::Processed.to_str(), "processed"); - assert_eq!(MediaFileOriginCategory::Upload.to_str(), "upload"); - } + mod mechanical_checks { + use super::*; - #[test] - fn test_from_str() { - assert_eq!(MediaFileOriginCategory::from_str("inference").unwrap(), MediaFileOriginCategory::Inference); - assert_eq!(MediaFileOriginCategory::from_str("processed").unwrap(), MediaFileOriginCategory::Processed); - assert_eq!(MediaFileOriginCategory::from_str("upload").unwrap(), MediaFileOriginCategory::Upload); - assert!(MediaFileOriginCategory::from_str("foo").is_err()); + #[test] + fn variant_length() { + use strum::IntoEnumIterator; + assert_eq!(MediaFileOriginCategory::all_variants().len(), MediaFileOriginCategory::iter().len()); + } + + #[test] + fn round_trip() { + for variant in MediaFileOriginCategory::all_variants() { + assert_eq!(variant, MediaFileOriginCategory::from_str(variant.to_str()).unwrap()); + assert_eq!(variant, MediaFileOriginCategory::from_str(&format!("{}", variant)).unwrap()); + assert_eq!(variant, MediaFileOriginCategory::from_str(&format!("{:?}", variant)).unwrap()); + } + } } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index b4064354fc..6721b683d5 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -26,6 +26,10 @@ pub enum MediaFileOriginProductCategory { /// uploaded on behalf of, etc.) #[serde(rename = "unknown")] Unknown, + + /// Media files created by (or uploaded for) the Zero Shot voice product. + #[serde(rename = "zs_voice")] + ZeroShotVoice, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -38,6 +42,7 @@ impl MediaFileOriginProductCategory { match self { Self::FaceAnimator => "face_animator", Self::Unknown => "unknown", + Self::ZeroShotVoice => "zs_voice", } } @@ -45,6 +50,7 @@ impl MediaFileOriginProductCategory { match value { "face_animator" => Ok(Self::FaceAnimator), "unknown" => Ok(Self::Unknown), + "zs_voice" => Ok(Self::ZeroShotVoice), _ => Err(format!("invalid value: {:?}", value)), } } @@ -55,6 +61,7 @@ impl MediaFileOriginProductCategory { BTreeSet::from([ Self::FaceAnimator, Self::Unknown, + Self::ZeroShotVoice, ]) } } @@ -71,26 +78,30 @@ mod tests { fn test_serialization() { assert_serialization(MediaFileOriginProductCategory::FaceAnimator, "face_animator"); assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); + assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); } #[test] fn to_str() { assert_eq!(MediaFileOriginProductCategory::FaceAnimator.to_str(), "face_animator"); assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); + assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); } #[test] fn from_str() { assert_eq!(MediaFileOriginProductCategory::from_str("face_animator").unwrap(), MediaFileOriginProductCategory::FaceAnimator); assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); + assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); } #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 2); + assert_eq!(variants.len(), 3); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/get_dataset_sample_by_uuid.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/get_dataset_sample_by_uuid.rs new file mode 100644 index 0000000000..3ccaebc5c0 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/get_dataset_sample_by_uuid.rs @@ -0,0 +1,67 @@ +use anyhow::anyhow; +use log::error; +use sqlx::{MySql, MySqlPool}; +use sqlx::pool::PoolConnection; + +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; + +pub struct DatasetSampleRecord { + pub token: ZsVoiceDatasetSampleToken, + pub media_file_token: MediaFileToken, +} + +/// Query for a media upload to see if we already uploaded it. +pub async fn get_dataset_sample_by_uuid( + uuid_idempotency_token: &str, + mysql_pool: &MySqlPool, +) -> AnyhowResult> { + let mut connection = mysql_pool.acquire().await?; + get_dataset_sample_by_uuid_with_connection( + uuid_idempotency_token, + &mut connection + ).await +} + +/// Query for a media upload to see if we already uploaded it. +pub async fn get_dataset_sample_by_uuid_with_connection( + uuid_idempotency_token: &str, + mysql_connection: &mut PoolConnection, +) -> AnyhowResult> { + let maybe_result = sqlx::query_as!( + RawDatasetSampleRecord, + r#" +SELECT + token as `token: tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken`, + media_file_token as `media_file_token: tokens::tokens::media_files::MediaFileToken` +FROM zs_voice_dataset_samples +WHERE + uuid_idempotency_token = ? + AND user_deleted_at IS NULL + AND mod_deleted_at IS NULL + "#, + uuid_idempotency_token, + ) + .fetch_one(mysql_connection) + .await; + + match maybe_result { + Err(err) => match err { + sqlx::Error::RowNotFound => Ok(None), + _ => { + error!("list media uploads db error: {:?}", err); + Err(anyhow!("error with query: {:?}", err)) + } + }, + Ok(upload) => Ok(Some(DatasetSampleRecord { + token: upload.token, + media_file_token: upload.media_file_token, + })) + } +} + +struct RawDatasetSampleRecord { + pub token: ZsVoiceDatasetSampleToken, + pub media_file_token: MediaFileToken, +} diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs new file mode 100644 index 0000000000..50676d74eb --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs @@ -0,0 +1,224 @@ +use anyhow::anyhow; +use log::warn; +use sqlx::MySqlPool; +use buckets::public::media_files::original_file::MediaFileBucketPath; + +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_uploads::media_upload_source::MediaUploadSource; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::media_uploads::MediaUploadToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; +use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; + +use crate::payloads::media_upload_modification_details::MediaUploadModificationDetails; + +/// Used to give user-facing order to logged in user inference requests +pub struct SyntheticIdRecord { + pub next_id: i64, +} + +pub struct Args <'a> { + pub uuid_idempotency_token: &'a str, + + // Dataset to which this sample belongs + pub dataset_token: ZsVoiceDatasetToken, + + pub media_type: MediaUploadType, + + pub origin_category: MediaFileOriginCategory, + + // TODO: + //pub maybe_original_filename: Option<&'a str>, + //pub original_file_size_bytes: u64, + //pub maybe_original_duration_millis: Option, + //pub maybe_original_mime_type: Option<&'a str>, + //pub maybe_original_audio_encoding: Option<&'a str>, + //pub maybe_original_video_encoding: Option<&'a str>, + //pub maybe_original_frame_width: Option, + //pub maybe_original_frame_height: Option, + + pub checksum_sha2: &'a str, + + pub media_file_path: &'a MediaFileBucketPath, + + pub maybe_creator_user_token: Option<&'a UserToken>, + pub maybe_creator_anonymous_visitor_token: Option<&'a str>, + pub creator_ip_address: &'a str, + pub creator_set_visibility: Visibility, + + pub mysql_pool: &'a MySqlPool, +} + +const ORIGIN_PRODUCT : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Unknown; // TODO: Zero shot voice enum + +pub async fn insert_dataset_sample_and_media_file(args: Args<'_>) -> AnyhowResult<(ZsVoiceDatasetSampleToken, MediaFileToken, u64)> { + + let mut maybe_creator_synthetic_id : Option = None; + + let mut transaction = args.mysql_pool.begin().await?; + +// TODO: Synthetic IDs + +// if let Some(creator_user_token) = args.maybe_creator_user_token.as_deref() { +// let query_result = sqlx::query!( +// r#" +//INSERT INTO media_upload_synthetic_ids +//SET +// user_token = ?, +// next_id = 1 +//ON DUPLICATE KEY UPDATE +// user_token = ?, +// next_id = next_id + 1 +// "#, +// creator_user_token, +// creator_user_token +// ) +// .execute(&mut transaction) +// .await; +// +// match query_result { +// Ok(_) => {}, +// Err(err) => { +// //transaction.rollback().await?; +// warn!("Transaction failure: {:?}", err); +// } +// } +// +// let query_result = sqlx::query_as!( +// SyntheticIdRecord, +// r#" +//SELECT +// next_id +//FROM +// media_upload_synthetic_ids +//WHERE +// user_token = ? +//LIMIT 1 +// "#, +// creator_user_token, +// ) +// .fetch_one(&mut transaction) +// .await; +// +// let record : SyntheticIdRecord = match query_result { +// Ok(record) => record, +// Err(err) => { +// warn!("Transaction failure: {:?}", err); +// transaction.rollback().await?; +// return Err(anyhow!("Transaction failure: {:?}", err)); +// } +// }; +// +// let next_id = record.next_id as u64; +// maybe_creator_synthetic_id = Some(next_id); +// } + + let media_file_token = MediaFileToken::generate(); + let dataset_sample_token = ZsVoiceDatasetSampleToken::generate(); + +// let maybe_extra_file_modification_info = args.maybe_extra_file_modification_info +// .clone() // FIXME +// .map(|info| info.to_json()) +// .transpose()?; + + let query = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + + media_type = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + creator_set_visibility = ? + + "#, + media_file_token, + + args.origin_category, + ORIGIN_PRODUCT, + + args.media_type, + + //args.maybe_original_filename, + //args.original_file_size_bytes, + //args.maybe_original_duration_millis.unwrap_or(0), + //args.maybe_original_mime_type, + //args.maybe_original_audio_encoding, + //args.maybe_original_video_encoding, + //args.maybe_original_frame_width, + //args.maybe_original_frame_height, + args.checksum_sha2, + + args.media_file_path.get_object_hash(), + + //maybe_extra_file_modification_info, + + args.maybe_creator_user_token, + //maybe_creator_synthetic_id, + args.maybe_creator_anonymous_visitor_token, + args.creator_ip_address, + args.creator_set_visibility.to_str() + ); + + let query_result = query.execute(&mut transaction) + .await; + + if let Err(err) = query_result { + return Err(anyhow!("error inserting new dataset sample: {:?}", err)); + } + + let query = sqlx::query!( + r#" + INSERT INTO zs_voice_dataset_samples SET + + token = ?, + + uuid_idempotency_token = ?, + + dataset_token = ?, + media_file_token = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ? + "#, + dataset_sample_token, + + args.uuid_idempotency_token, + + args.dataset_token, + media_file_token, + + args.maybe_creator_user_token, + args.maybe_creator_anonymous_visitor_token, + args.creator_ip_address, + ); + + let query_result = query.execute(&mut transaction) + .await; + + let result_tuple = match query_result { + Ok(res) => (dataset_sample_token, media_file_token, res.last_insert_id()), + Err(err) => return Err(anyhow!("error inserting new dataset sample: {:?}", err)), + }; + + transaction.commit().await?; + + Ok(result_tuple) +} diff --git a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/mod.rs b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/mod.rs index 42210d73f5..e5bce36894 100644 --- a/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/mod.rs +++ b/crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/mod.rs @@ -1,3 +1,5 @@ pub mod delete_sample; pub mod get_dataset_sample; +pub mod get_dataset_sample_by_uuid; +pub mod insert_dataset_sample_and_media_file; pub mod list_dataset_samples; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/drain_multipart_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/drain_multipart_request.rs new file mode 100644 index 0000000000..3f6573ecba --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/drain_multipart_request.rs @@ -0,0 +1,89 @@ +use actix_multipart::Multipart; +use actix_web::web::BytesMut; +use futures::TryStreamExt; +use log::warn; + +use errors::AnyhowResult; + +use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; + +pub struct UploadMediaRequest { + pub uuid_idempotency_token: Option, + pub file_name: Option, + pub file_bytes: Option, + pub media_source: MediaSource, +} + +/// Where the frontend tells us the file came from. +#[derive(Eq, PartialEq, Clone, Copy)] +pub enum MediaSource { + Unknown, + UserFile, + /// Eg. the web audio API in Javascript. + UserDeviceApi, +} + +/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. +pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { + let mut uuid_idempotency_token = None; + let mut file_bytes = None; + let mut file_name = None; + let mut media_source = None; + + while let Ok(Some(mut field)) = multipart_payload.try_next().await { + let mut field_name = None; + let mut field_filename = None; + + if let Some(content_disposition) = field.content_disposition() { + field_name = content_disposition.get_name() + .map(|s| s.to_string()); + field_filename = content_disposition.get_filename() // NB: Only used for the file bytes. + .map(|s| s.to_string()); + } + + match field_name.as_deref() { + Some("uuid_idempotency_token") => { + uuid_idempotency_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading idempotency token: {:}", e); + e + })?; + }, + Some("file") => { + file_name = field_filename.clone(); + file_bytes = checked_read_multipart_bytes(&mut field).await + .map_err(|e| { + warn!("Error reading audio upload: {:}", e); + e + })?; + }, + Some("source") => { + media_source = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading source: {:}", e); + e + })?; + }, + _ => continue, + } + } + + let media_source = match media_source.as_deref() { + Some("device") => MediaSource::UserDeviceApi, + Some("file") => MediaSource::UserFile, + _ => { + if file_name.as_deref() == Some("blob") { + MediaSource::UserDeviceApi + } else { + MediaSource::Unknown + } + }, + }; + + Ok(UploadMediaRequest { + uuid_idempotency_token, + file_name, + file_bytes, + media_source, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index e320d83853..a965ca7b81 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -1 +1,2 @@ +pub mod drain_multipart_request; pub mod get_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/drain_multipart_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/drain_multipart_request.rs index 3f6573ecba..755988b96c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/drain_multipart_request.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/common/drain_multipart_request.rs @@ -4,11 +4,13 @@ use futures::TryStreamExt; use log::warn; use errors::AnyhowResult; +use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; -pub struct UploadMediaRequest { +pub struct UploadSampleRequest { pub uuid_idempotency_token: Option, + pub dataset_token: Option, pub file_name: Option, pub file_bytes: Option, pub media_source: MediaSource, @@ -24,8 +26,9 @@ pub enum MediaSource { } /// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. -pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { +pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { let mut uuid_idempotency_token = None; + let mut dataset_token = None; let mut file_bytes = None; let mut file_name = None; let mut media_source = None; @@ -49,6 +52,13 @@ pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> Anyhow e })?; }, + Some("dataset_token") => { + dataset_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading dataset token: {:}", e); + e + })?; + }, Some("file") => { file_name = field_filename.clone(); file_bytes = checked_read_multipart_bytes(&mut field).await @@ -80,8 +90,9 @@ pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> Anyhow }, }; - Ok(UploadMediaRequest { + Ok(UploadSampleRequest { uuid_idempotency_token, + dataset_token, file_name, file_bytes, media_source, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs index b45e7b550c..ce41db04e6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs @@ -27,7 +27,7 @@ static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { "audio/x-flac", "audio/x-wav", - // This might be the only way to allow browser recording + // NB(bt,2023-10-13): This is the only way to allow browser recording. // https://air.ghost.io/recording-to-an-audio-file-using-html5-and-js/ // Chrome: "audio/webm;codecs=opus" // Firefox: "audio/ogg;codecs=opus" diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_sample.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_sample.rs index 6054266484..b99f392552 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_sample.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_sample.rs @@ -1,15 +1,381 @@ - - +use std::collections::HashSet; use std::sync::Arc; -use actix_web::{HttpRequest, HttpResponse, web}; -use actix_web::error::ResponseError; -use actix_web::http::StatusCode; +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; +use hyper::StatusCode; +use log::{error, info, warn}; +use once_cell::sync::Lazy; +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType::MediaUpload; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_uploads::media_upload_type::MediaUploadType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; -use log::{info, warn}; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_uploads::insert_media_upload::insert_media_upload; +use mysql_queries::queries::voice_designer::voice_samples::get_dataset_sample_by_uuid::get_dataset_sample_by_uuid_with_connection; +use mysql_queries::queries::voice_designer::voice_samples::insert_dataset_sample_and_media_file::{Args, insert_dataset_sample_and_media_file}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken; +use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; +use crate::http_server::endpoints::media_uploads::common::drain_multipart_request::{drain_multipart_request, MediaSource}; +use crate::server_state::ServerState; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; -pub async fn upload_sample_handler() -> HttpResponse { - // Implementation for uploading a sample - HttpResponse::Ok().json("Sample uploaded successfully") -} \ No newline at end of file +#[derive(Serialize)] +pub struct UploadSampleResponse { + pub success: bool, + pub sample_token: ZsVoiceDatasetSampleToken, + pub media_file_token: MediaFileToken, +} + +#[derive(Debug, Serialize)] +pub enum UploadSampleError { + BadInput(String), + NotAuthorized, + MustBeLoggedIn, + ServerError, + RateLimited, +} + +impl ResponseError for UploadSampleError { + fn status_code(&self) -> StatusCode { + match *self { + UploadSampleError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadSampleError::NotAuthorized => StatusCode::UNAUTHORIZED, + UploadSampleError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadSampleError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadSampleError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl std::fmt::Display for UploadSampleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +static ALLOWED_AUDIO_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + "audio/aac", + "audio/m4a", + "audio/mpeg", + "audio/ogg", + "audio/opus", + "audio/x-flac", + "audio/x-wav", + + // NB(bt,2023-10-13): This is the only way to allow browser recording. + // https://air.ghost.io/recording-to-an-audio-file-using-html5-and-js/ + // Chrome: "audio/webm;codecs=opus" + // Firefox: "audio/ogg;codecs=opus" + "audio/webm", + "video/webm", + ]) +}); +pub async fn upload_sample_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart, +) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + UploadSampleError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + UploadSampleError::ServerError + })?; + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.is_banned { + return Err(UploadSampleError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(UploadSampleError::RateLimited); + } + + if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { + return Err(UploadSampleError::RateLimited); + } + + // ==================== READ MULTIPART REQUEST ==================== // + + let upload_sample_request = drain_multipart_request(multipart_payload) + .await + .map_err(|e| { + // TODO: Error handling could be nicer. + UploadSampleError::BadInput("bad request".to_string()) + })?; + + let uuid_idempotency_token = upload_sample_request.uuid_idempotency_token + .ok_or(UploadSampleError::BadInput("no uuid".to_string()))?; + + let maybe_existing_upload = + get_dataset_sample_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) + .await; + + match maybe_existing_upload { + Err(err) => { + error!("Error checking for previous upload: {:?}", err); + return Err(UploadSampleError::ServerError); + } + Ok(Some(previous_record)) => { + return success_response( + previous_record.token, + previous_record.media_file_token, + ); + } + Ok(None) => { + // Proceed. + } + } + + if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { + return Err(UploadSampleError::BadInput(reason)); + } + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) + .unwrap_or(Visibility::default()); + + let ip_address = get_request_ip(&http_request); + + let maybe_user_token = maybe_user_session + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_file_size_bytes = upload_sample_request.file_bytes + .as_ref() + .map(|bytes| bytes.len()); + + let maybe_mimetype = upload_sample_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + + let dataset_token = match upload_sample_request.dataset_token { + None => return Err(UploadSampleError::BadInput("missing dataset token".to_string())), + Some(token) => ZsVoiceDatasetToken::new_from_str(&token), + }; + + let bytes = match upload_sample_request.file_bytes { + None => return Err(UploadSampleError::BadInput("missing file contents".to_string())), + Some(bytes) => bytes, + }; + + let hash = sha256_hash_bytes(&bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + UploadSampleError::ServerError + })?; + + let file_size_bytes = bytes.len(); + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + let mut media_upload_type = None; + + if let Some(mimetype) = maybe_mimetype.as_deref() { + + if !ALLOWED_AUDIO_MIME_TYPES.contains(mimetype) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(UploadSampleError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // NB: .aiff (audio/aiff) isn't supported by Symphonia: + // It contains uncompressed PCM-encoded audio similar to wav. + // See: https://github.com/pdeljanov/Symphonia/issues/75 + // NB: The following formats are not supported by Symphonia and + // do not have any open issues filed. They may simply be too old: + // - .wma (audio/x-ms-wma) + // - .avi (video/x-msvideo) + media_upload_type = match mimetype { + // Audio + "audio/aac" /* .aac */ => Some(MediaUploadType::Audio), + "audio/m4a" /* .m4a */ => Some(MediaUploadType::Audio), + "audio/mpeg" /* .mp3 */ => Some(MediaUploadType::Audio), + "audio/ogg" /* .ogg */ => Some(MediaUploadType::Audio), + "audio/opus" /* .opus */ => Some(MediaUploadType::Audio), + "audio/x-flac" /* .flac */ => Some(MediaUploadType::Audio), + "audio/x-wav" /* .wav */ => Some(MediaUploadType::Audio), + // Video + "video/webm" /* .webm */ => Some(MediaUploadType::Video), // TODO: Do browsers send this for audio (?) + _ => None, + }; + + let do_audio_decode = match mimetype { + "audio/opus" => { + // TODO/FIXME(bt, 2023-05-19): Symphonia is currently broken for Firefox's opus. + // We're on an off-master branch that may resolve the problem in the future, but for now + // it panics as follows: + // + // [2023-05-19T10:25:34Z INFO symphonia_core::probe] found a possible format marker within [4f, 67, 67, 53, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, d4, c5] @ 0+2 bytes. + // [2023-05-19T10:25:34Z INFO symphonia_core::probe] found the format marker [4f, 67, 67, 53] @ 0+2 bytes. + // [2023-05-19T10:25:34Z DEBUG symphonia_format_ogg::page] grow page buffer to 8192 bytes + // [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] starting new physical stream + // [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] selected opus mapper for stream with serial=0x19aac5d4 + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Probed! + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Find audio track... + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Found audio track (maybe) + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe track duration: None + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe codec short name: Some("opus") + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Opus handler + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Media source stream + // [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : options... + // [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : try_read... + // [2023-05-19T10:25:34Z DEBUG symphonia_format_mkv::ebml] element with tag: 4F67 + // thread 'actix-rt|system:0|arbiter:1' panicked at 'assertion failed: `(left == right)` + // left: `Unknown`, + // right: `Ebml`: EBML element type must be checked before calling this function', /Users/bt/.cargo/git/checkouts/symphonia-8fbe6c90fc095688/e1a7009/symphonia-format-mkv/src/ebml.rs:335:9 + false + } + // Also, don't decode images + "image/jpeg" => false, + "image/png" => false, + "image/webp" => false, + _ => true, + }; + + if do_audio_decode && media_upload_type.is_some() { + let basic_info = decode_basic_audio_bytes_info( + bytes.as_ref(), + Some(mimetype), + None + ).map_err(|e| { + warn!("file decoding error: {:?}", e); + UploadSampleError::BadInput("could not decode file".to_string()) + })?; + + maybe_duration_millis = basic_info.duration_millis; + maybe_codec_name = basic_info.codec_name; + } + } + + let media_upload_type = match media_upload_type { + Some(m) => m, + None => { + warn!("Invalid mimetype: {:?}", maybe_mimetype); + return Err(UploadSampleError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype))) + }, + }; + + let media_file_origin = match upload_sample_request.media_source { + MediaSource::Unknown => MediaFileOriginCategory::Upload, // NB: Just treat as a regular upload. + MediaSource::UserFile => MediaFileOriginCategory::Upload, + MediaSource::UserDeviceApi => MediaFileOriginCategory::DeviceApi, + }; + + // TODO: Clean up code + let mime_type = match maybe_mimetype { + Some(m) => m, + None => { + warn!("Missing mimetype!"); + return Err(UploadSampleError::BadInput("Missing mimetype".to_string())); + }, + }; + + let public_upload_path = MediaFileBucketPath::generate_new(Some("sample_"), Some(".bin")); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + UploadSampleError::ServerError + })?; + + let (dataset_sample_token, media_file_token, _record_id) = insert_dataset_sample_and_media_file(Args { + uuid_idempotency_token: &uuid_idempotency_token, + media_type: media_upload_type, + origin_category: media_file_origin, + dataset_token: dataset_token, + //maybe_original_filename: upload_media_request.file_name.as_deref(), + //original_file_size_bytes: file_size_bytes as u64, + //maybe_original_duration_millis: maybe_duration_millis, + //maybe_original_mime_type: maybe_mimetype, + //maybe_original_audio_encoding: maybe_codec_name.as_deref(), + //maybe_original_video_encoding: None, + //maybe_original_frame_width: None, // TODO + //maybe_original_frame_height: None, // TODO + checksum_sha2: &hash, + media_file_path: &public_upload_path, + //maybe_extra_file_modification_info: None, + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: None, + creator_ip_address: &ip_address, + creator_set_visibility, + mysql_pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New generic download creation DB error: {:?}", err); + UploadSampleError::ServerError + })?; + + info!("new media file token: {:?} dataset token: {:?}", media_file_token, dataset_sample_token); + + success_response( + dataset_sample_token, + media_file_token, + ) +} + +fn success_response( + sample_token: ZsVoiceDatasetSampleToken, + media_file_token: MediaFileToken +) -> Result { + + let response = UploadSampleResponse { + success: true, + sample_token, + media_file_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| UploadSampleError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index bbd27aa871..15f2995719 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -172,7 +172,7 @@ pub fn add_routes (app: App) -> App // TODO find a long term feature flag solution, since this code is likely deployed into production we don't want the route found. let enable_voice_designer_route = false; - if (enable_voice_designer_route) { + if enable_voice_designer_route { app = add_voice_designer_routes(app); /* /v1/voice_designer */ } // ==================== Comments ====================