diff --git a/_sql/migrations/2023-09-26-070239_zs_voices/down.sql b/_sql/migrations/2023-09-26-070239_zs_voices/down.sql new file mode 100644 index 0000000000..46e0bb8f7e --- /dev/null +++ b/_sql/migrations/2023-09-26-070239_zs_voices/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table zs_voices; diff --git a/_sql/migrations/2023-09-26-070239_zs_voices/up.sql b/_sql/migrations/2023-09-26-070239_zs_voices/up.sql new file mode 100644 index 0000000000..7b1beb4c95 --- /dev/null +++ b/_sql/migrations/2023-09-26-070239_zs_voices/up.sql @@ -0,0 +1,154 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- This table contains both zero shot TTS voices (speaker embeddings) and zero shot V2V voices (speaker embeddings). +-- These are the zero shot voices that are usable by our users. +CREATE TABLE zs_voices ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" for joins, API, etc. (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- NB: No idempotency token. + -- Store it in another table to prevent duplicate record creation from rapid user inputs. + + -- ========== ZERO SHOT FOREIGN KEYS AND STATE========== + + -- The zero shot dataset that was used to create this voice. + -- The dataset will only be accessible to the original author of the voice. + -- The original author can create multiple voices from the same dataset. + dataset_token VARCHAR(32) NOT NULL, + + -- When datasets are edited, we bump their version. + -- This tells us which version of the dataset created this voice (not that we should preserve + -- record of which samples are in which version.) + -- We can surface this to the users as well. + dataset_version INT NOT NULL DEFAULT 0, + + -- ========== MODEL DETAILS ========== + + -- The class or category of model/algorithm this voice is intended for. + -- * 'tts' for zero shot TTS + -- * 'vc' for zero shot voice conversion + -- * 'pitch' for pitch correction models (hypothetical) + -- * (Since this isn't an enum, the list of values can be expanded.) + model_category VARCHAR(16) NOT NULL, + + -- The model used for the voice: + -- * 'vall-e-x' for Vall-E-X + -- * 'ns2vc' for NaturalSpeech2 voice conversion (hypothetical) + -- * (Since this isn't an enum, the list of values can be expanded.) + model_type VARCHAR(16) NOT NULL, + + -- The version of the model. + -- If we deploy new model weights for the model, we may need to regenerate + -- all of the corresponding speaker embeddings. This lets us know which voices + -- are the old version. + -- In some cases, we may need to preserve, hide, or roll back to older versions. + model_version INTEGER UNSIGNED NOT NULL DEFAULT 0, + + -- What kind of features these are: spectrogram, encodec, etc. + -- Technically we shouldn't need this, as the 2-tuple (model_type, model_version) implicitly knows + -- what its features should be, but this is probably useful for our own analysis and may be easier + -- to deal with. + -- + -- Example values: + -- * 'spectrogram' + -- * 'encodec' + -- * (Since this isn't an enum, the list of values can be expanded.) + encoding_type VARCHAR(16) NOT NULL, + + -- ========== VOICE DETAILS ========== + + -- The "title" of the voice. This can be the name and additional information as supplied by the user. + title VARCHAR(255) NOT NULL, + + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela" + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en") + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + + -- TODO: misc speaker metadata + -- We'll want rich metadata for search + -- series_name VARCHAR(255) DEFAULT NULL, + -- speaker_name VARCHAR(255) DEFAULT NULL, + -- speaker_gender VARCHAR(255) DEFAULT NULL, + -- speaker_age VARCHAR(255) DEFAULT NULL, + -- speaker_voice_style VARCHAR(255) DEFAULT NULL, + + -- ========== STORAGE DETAILS ========== + + -- The hash for the bucket directory that contains the original upload + -- eg. /speaker_vectors/{bucket_hash}/{voice_model_version}.bin + bucket_hash VARCHAR(32) NOT NULL, + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- The creator can set a desired visibility for their data. + -- This does not always apply to every upload type. + -- Additionally, some upload types may require moderator approval prior + -- to being publicly listed, and this field has no bearing on that. + -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes last. + -- Fine details of moderator actions are kept in a separate `audit_logs` table. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + + -- TODO: These are not correct. Fix these before landing / running migrations. + KEY index_model_category (model_category), + KEY index_model_type (model_type), + KEY index_model_type_and_version (model_type, model_version), + KEY index_encoding_type (encoding_type), + KEY index_ietf_primary_language_subtag (ietf_primary_language_subtag), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at (user_deleted_at), + KEY index_mod_deleted_at (mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations/2023-09-26-070244_zs_voice_datasets/down.sql b/_sql/migrations/2023-09-26-070244_zs_voice_datasets/down.sql new file mode 100644 index 0000000000..5b5863138a --- /dev/null +++ b/_sql/migrations/2023-09-26-070244_zs_voice_datasets/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table zs_voice_datasets; diff --git a/_sql/migrations/2023-09-26-070244_zs_voice_datasets/up.sql b/_sql/migrations/2023-09-26-070244_zs_voice_datasets/up.sql new file mode 100644 index 0000000000..9fb2edd8f4 --- /dev/null +++ b/_sql/migrations/2023-09-26-070244_zs_voice_datasets/up.sql @@ -0,0 +1,106 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- This table describes a zero shot voice in the process of being created (or later reused / edited / enhanced) +-- These are not usable voices until their vector embeddings have been created, and this table does not serve +-- the production voice list. +-- +-- Multiple voices can be created from a single "dataset" as the data set is mutable over time. Users can add or +-- remove new samples to the data set at any time. +CREATE TABLE zs_voice_datasets ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" for joins, API, etc. (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- NB: No idempotency token. + -- Store it in another table to prevent duplicate record creation from rapid user inputs. + + -- ========== VOICE DETAILS ========== + + -- The "title" of the voice. This can be the name and additional information as supplied by the user. + -- When the voice embeddings are created, this will be copied to the produced "voice" (embeddings) record. + title VARCHAR(255) NOT NULL, + + -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela" + ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', + + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en") + ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', + + -- TODO: misc speaker metadata + -- We'll want rich metadata for search + -- When the voice embeddings are created, this will be copied to the produced "voice" (embeddings) record. + -- We don't want a hard link between the dataset and the produced voice as the voice is static once + -- created (with the exception of regenerating embeddings), but the dataset my drift as the user experiments, + -- adds/removes samples, etc. + -- series_name VARCHAR(255) DEFAULT NULL, + -- speaker_name VARCHAR(255) DEFAULT NULL, + -- speaker_gender VARCHAR(255) DEFAULT NULL, + -- speaker_age VARCHAR(255) DEFAULT NULL, + -- speaker_voice_style VARCHAR(255) DEFAULT NULL, + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- The creator can set a desired visibility for their data. + -- This does not always apply to every upload type. + -- Additionally, some upload types may require moderator approval prior + -- to being publicly listed, and this field has no bearing on that. + -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS. + creator_set_visibility ENUM( + 'public', + 'hidden', + 'private' + ) NOT NULL DEFAULT 'public', + + -- The synthetic id associated with this result. + -- These ids are incremented on a per-user basis to help users + -- sequence their own work. They serve no other purpose. + maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes last. + -- Fine details of moderator actions are kept in a separate `audit_logs` table. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + + KEY index_ietf_primary_language_subtag (ietf_primary_language_subtag), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at (user_deleted_at), + KEY index_mod_deleted_at (mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/down.sql b/_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/down.sql new file mode 100644 index 0000000000..b161d29075 --- /dev/null +++ b/_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table zs_voice_dataset_samples; diff --git a/_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/up.sql b/_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/up.sql new file mode 100644 index 0000000000..f9b387095a --- /dev/null +++ b/_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/up.sql @@ -0,0 +1,84 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- These are the individual files in each dataset. +-- Files can be added and removed from a dataset at any time. +CREATE TABLE zs_voice_dataset_samples ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" for joins, API, etc. (PUBLIC) + token VARCHAR(32) NOT NULL, + + -- NB: No idempotency token. + -- Store it in another table to prevent duplicate record creation from rapid user inputs. + + -- ========== ZERO SHOT FOREIGN KEYS AND STATE========== + + -- The zero shot dataset that was used to create this voice. + -- The dataset will only be accessible to the original author of the voice. + -- The original author can create multiple voices from the same dataset. + dataset_token VARCHAR(32) NOT NULL, + + -- Foreign key to the uploaded media file. + -- There was an option of handling uploads and file management separately, but by electing to store the dataset + -- samples in the "media_files" table, they can be reused by our other code, queries, machinery, etc. + -- Note that this requires that the "media_files" record must be an audio (not video) file (which our code will + -- enforce), and that it is given a category of "voice_zero_shot" so it isn't co-mingled with other uploads/outputs + media_file_token VARCHAR(32) NOT NULL, + + -- Some data sets require raw transcripts for the audio. + -- If required, they'll live here. + maybe_raw_transcription TEXT DEFAULT NULL, + + -- A way to order the data set samples + -- We don't necessarily need to reimplement file reordering. + -- order_id INT DEFAULT NULL, + + -- ========== CREATOR DETAILS AND PREFERENCES ========== + + -- Foreign key to user + maybe_creator_user_token VARCHAR(32) DEFAULT NULL, + + -- Based on a cookie sent by the frontend. + -- We'll save this even if the user is logged in. + maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL, + + -- For abuse tracking. + -- Wide enough for IPv4/6 + creator_ip_address VARCHAR(40) NOT NULL, + + -- ========== MODERATION DETAILS ========== + + -- The last moderator that made changes last. + -- Fine details of moderator actions are kept in a separate `audit_logs` table. + maybe_mod_user_token VARCHAR(32) DEFAULT NULL, + + -- ========== TIMESTAMPS ========== + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- If this is removed by a mod or the creator. + user_deleted_at TIMESTAMP NULL, + mod_deleted_at TIMESTAMP NULL, + + -- ========== VECTOR CLOCK ========== + + -- Incremented with every update. + version INT NOT NULL DEFAULT 0, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (token), + + KEY fk_dataset_token (dataset_token), + KEY fk_media_file_token (media_file_token), + KEY fk_maybe_creator_user_token (maybe_creator_user_token), + KEY fk_maybe_mod_user_token (maybe_mod_user_token), + KEY index_creator_ip_address (creator_ip_address), + KEY index_user_deleted_at (user_deleted_at), + KEY index_mod_deleted_at (mod_deleted_at) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations/2023-09-29-033922_zs_voice_included_samples/down.sql b/_sql/migrations/2023-09-29-033922_zs_voice_included_samples/down.sql new file mode 100644 index 0000000000..6da78ad9c0 --- /dev/null +++ b/_sql/migrations/2023-09-29-033922_zs_voice_included_samples/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +drop table zs_voice_included_samples; diff --git a/_sql/migrations/2023-09-29-033922_zs_voice_included_samples/up.sql b/_sql/migrations/2023-09-29-033922_zs_voice_included_samples/up.sql new file mode 100644 index 0000000000..abf45704ad --- /dev/null +++ b/_sql/migrations/2023-09-29-033922_zs_voice_included_samples/up.sql @@ -0,0 +1,26 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- In order to rebuild any "zs_voice", we need to know which dataset samples were included. +-- Since users can change the dataset at any time after a voice was originally created, these +-- records are generated to retain a record of which samples fed into a given voice. +-- These records will probably not be used for any product features, just as a means to study +-- poor-performing voices, perform migrations, etc. +CREATE TABLE zs_voice_included_samples ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- First member of composite effective "primary key" + -- The "zs_voice" that was created. + voice_token VARCHAR(32) NOT NULL, + + -- Second member of composite effective "primary key" + -- The "zs_voice_dataset_sample" that was used. + dataset_sample_token VARCHAR(32) NOT NULL, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (voice_token, dataset_sample_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/_sql/migrations_squashed/tts_tables/tts_models.sql b/_sql/migrations_squashed/tts_tables/tts_models.sql index af3f6786dd..493d2c3cd2 100644 --- a/_sql/migrations_squashed/tts_tables/tts_models.sql +++ b/_sql/migrations_squashed/tts_tables/tts_models.sql @@ -83,9 +83,10 @@ CREATE TABLE tts_models ( -- ========== UNSTRUCTURED METADATA ========== -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.) + -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela" ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en', - -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es") + -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en") ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en', -- The "name" of the voice model, which might be complicated.