Zero shot tables

_sql/migrations/2023-09-26-070239_zs_voices/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+drop table zs_voices;

_sql/migrations/2023-09-26-070239_zs_voices/up.sql

@@ -0,0 +1,154 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- This table contains both zero shot TTS voices (speaker embeddings) and zero shot V2V voices (speaker embeddings).
+-- These are the zero shot voices that are usable by our users.
+CREATE TABLE zs_voices (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" for joins, API, etc. (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- NB: No idempotency token.
+  -- Store it in another table to prevent duplicate record creation from rapid user inputs.
+
+  -- ========== ZERO SHOT FOREIGN KEYS AND STATE==========
+
+  -- The zero shot dataset that was used to create this voice.
+  -- The dataset will only be accessible to the original author of the voice.
+  -- The original author can create multiple voices from the same dataset.
+  dataset_token VARCHAR(32) NOT NULL,
+
+  -- When datasets are edited, we bump their version.
+  -- This tells us which version of the dataset created this voice (not that we should preserve
+  -- record of which samples are in which version.)
+  -- We can surface this to the users as well.
+  dataset_version INT NOT NULL DEFAULT 0,
+
+  -- ========== MODEL DETAILS ==========
+
+  -- The class or category of model/algorithm this voice is intended for.
+  --   * 'tts' for zero shot TTS
+  --   * 'vc' for zero shot voice conversion
+  --   * 'pitch' for pitch correction models (hypothetical)
+  --   * (Since this isn't an enum, the list of values can be expanded.)
+  model_category VARCHAR(16) NOT NULL,
+
+  -- The model used for the voice:
+  --   * 'vall-e-x' for Vall-E-X
+  --   * 'ns2vc' for NaturalSpeech2 voice conversion (hypothetical)
+  --   * (Since this isn't an enum, the list of values can be expanded.)
+  model_type VARCHAR(16) NOT NULL,
+
+  -- The version of the model.
+  -- If we deploy new model weights for the model, we may need to regenerate
+  -- all of the corresponding speaker embeddings. This lets us know which voices
+  -- are the old version.
+  -- In some cases, we may need to preserve, hide, or roll back to older versions.
+  model_version INTEGER UNSIGNED NOT NULL DEFAULT 0,
+
+  -- What kind of features these are: spectrogram, encodec, etc.
+  -- Technically we shouldn't need this, as the 2-tuple (model_type, model_version) implicitly knows
+  -- what its features should be, but this is probably useful for our own analysis and may be easier
+  -- to deal with.
+  --
+  -- Example values:
+  --   * 'spectrogram'
+  --   * 'encodec'
+  --   * (Since this isn't an enum, the list of values can be expanded.)
+  encoding_type VARCHAR(16) NOT NULL,
+
+  -- ========== VOICE DETAILS ==========
+
+  -- The "title" of the voice. This can be the name and additional information as supplied by the user.
+  title VARCHAR(255) NOT NULL,
+
+  -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
+  -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela"
+  ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
+
+  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en")
+  ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
+
+  -- TODO: misc speaker metadata
+  -- We'll want rich metadata for search
+  -- series_name VARCHAR(255) DEFAULT NULL,
+  -- speaker_name VARCHAR(255) DEFAULT NULL,
+  -- speaker_gender VARCHAR(255) DEFAULT NULL,
+  -- speaker_age VARCHAR(255) DEFAULT NULL,
+  -- speaker_voice_style VARCHAR(255) DEFAULT NULL,
+
+  -- ========== STORAGE DETAILS ==========
+
+  -- The hash for the bucket directory that contains the original upload
+  -- eg. /speaker_vectors/{bucket_hash}/{voice_model_version}.bin
+  bucket_hash  VARCHAR(32) NOT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- Based on a cookie sent by the frontend.
+  -- We'll save this even if the user is logged in.
+  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- The creator can set a desired visibility for their data.
+  -- This does not always apply to every upload type.
+  -- Additionally, some upload types may require moderator approval prior
+  -- to being publicly listed, and this field has no bearing on that.
+  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- The synthetic id associated with this result.
+  -- These ids are incremented on a per-user basis to help users
+  -- sequence their own work. They serve no other purpose.
+  maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- The last moderator that made changes last.
+  -- Fine details of moderator actions are kept in a separate `audit_logs` table.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod or the creator.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+
+  -- TODO: These are not correct. Fix these before landing / running migrations.
+  KEY index_model_category (model_category),
+  KEY index_model_type (model_type),
+  KEY index_model_type_and_version (model_type, model_version),
+  KEY index_encoding_type (encoding_type),
+  KEY index_ietf_primary_language_subtag (ietf_primary_language_subtag),
+  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
+  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
+  KEY index_creator_ip_address (creator_ip_address),
+  KEY index_user_deleted_at (user_deleted_at),
+  KEY index_mod_deleted_at (mod_deleted_at)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2023-09-26-070244_zs_voice_datasets/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+drop table zs_voice_datasets;

_sql/migrations/2023-09-26-070244_zs_voice_datasets/up.sql

@@ -0,0 +1,106 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- This table describes a zero shot voice in the process of being created (or later reused / edited / enhanced)
+-- These are not usable voices until their vector embeddings have been created, and this table does not serve
+-- the production voice list.
+--
+-- Multiple voices can be created from a single "dataset" as the data set is mutable over time. Users can add or
+-- remove new samples to the data set at any time.
+CREATE TABLE zs_voice_datasets (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" for joins, API, etc. (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- NB: No idempotency token.
+  -- Store it in another table to prevent duplicate record creation from rapid user inputs.
+
+  -- ========== VOICE DETAILS ==========
+
+  -- The "title" of the voice. This can be the name and additional information as supplied by the user.
+  -- When the voice embeddings are created, this will be copied to the produced "voice" (embeddings) record.
+  title VARCHAR(255) NOT NULL,
+
+  -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
+  -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela"
+  ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
+
+  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en")
+  ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
+
+  -- TODO: misc speaker metadata
+  -- We'll want rich metadata for search
+  -- When the voice embeddings are created, this will be copied to the produced "voice" (embeddings) record.
+  -- We don't want a hard link between the dataset and the produced voice as the voice is static once
+  -- created (with the exception of regenerating embeddings), but the dataset my drift as the user experiments,
+  -- adds/removes samples, etc.
+  -- series_name VARCHAR(255) DEFAULT NULL,
+  -- speaker_name VARCHAR(255) DEFAULT NULL,
+  -- speaker_gender VARCHAR(255) DEFAULT NULL,
+  -- speaker_age VARCHAR(255) DEFAULT NULL,
+  -- speaker_voice_style VARCHAR(255) DEFAULT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- Based on a cookie sent by the frontend.
+  -- We'll save this even if the user is logged in.
+  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- The creator can set a desired visibility for their data.
+  -- This does not always apply to every upload type.
+  -- Additionally, some upload types may require moderator approval prior
+  -- to being publicly listed, and this field has no bearing on that.
+  -- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
+  creator_set_visibility ENUM(
+    'public',
+    'hidden',
+    'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- The synthetic id associated with this result.
+  -- These ids are incremented on a per-user basis to help users
+  -- sequence their own work. They serve no other purpose.
+  maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- The last moderator that made changes last.
+  -- Fine details of moderator actions are kept in a separate `audit_logs` table.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod or the creator.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+
+  KEY index_ietf_primary_language_subtag (ietf_primary_language_subtag),
+  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
+  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
+  KEY index_creator_ip_address (creator_ip_address),
+  KEY index_user_deleted_at (user_deleted_at),
+  KEY index_mod_deleted_at (mod_deleted_at)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+drop table zs_voice_dataset_samples;

_sql/migrations/2023-09-26-070252_zs_voice_dataset_samples/up.sql

@@ -0,0 +1,84 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- These are the individual files in each dataset.
+-- Files can be added and removed from a dataset at any time.
+CREATE TABLE zs_voice_dataset_samples (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- Effective "primary key" for joins, API, etc. (PUBLIC)
+  token VARCHAR(32) NOT NULL,
+
+  -- NB: No idempotency token.
+  -- Store it in another table to prevent duplicate record creation from rapid user inputs.
+
+  -- ========== ZERO SHOT FOREIGN KEYS AND STATE==========
+
+  -- The zero shot dataset that was used to create this voice.
+  -- The dataset will only be accessible to the original author of the voice.
+  -- The original author can create multiple voices from the same dataset.
+  dataset_token VARCHAR(32) NOT NULL,
+
+  -- Foreign key to the uploaded media file.
+  -- There was an option of handling uploads and file management separately, but by electing to store the dataset
+  -- samples in the "media_files" table, they can be reused by our other code, queries, machinery, etc.
+  -- Note that this requires that the "media_files" record must be an audio (not video) file (which our code will
+  -- enforce), and that it is given a category of "voice_zero_shot" so it isn't co-mingled with other uploads/outputs
+  media_file_token VARCHAR(32) NOT NULL,
+
+  -- Some data sets require raw transcripts for the audio.
+  -- If required, they'll live here.
+  maybe_raw_transcription TEXT DEFAULT NULL,
+
+  -- A way to order the data set samples
+  -- We don't necessarily need to reimplement file reordering.
+  -- order_id INT DEFAULT NULL,
+
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- Based on a cookie sent by the frontend.
+  -- We'll save this even if the user is logged in.
+  maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- ========== MODERATION DETAILS ==========
+
+  -- The last moderator that made changes last.
+  -- Fine details of moderator actions are kept in a separate `audit_logs` table.
+  maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
+
+  -- ========== TIMESTAMPS ==========
+
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+
+  -- If this is removed by a mod or the creator.
+  user_deleted_at TIMESTAMP NULL,
+  mod_deleted_at TIMESTAMP NULL,
+
+  -- ========== VECTOR CLOCK ==========
+
+  -- Incremented with every update.
+  version INT NOT NULL DEFAULT 0,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (token),
+
+  KEY fk_dataset_token (dataset_token),
+  KEY fk_media_file_token (media_file_token),
+  KEY fk_maybe_creator_user_token (maybe_creator_user_token),
+  KEY fk_maybe_mod_user_token (maybe_mod_user_token),
+  KEY index_creator_ip_address (creator_ip_address),
+  KEY index_user_deleted_at (user_deleted_at),
+  KEY index_mod_deleted_at (mod_deleted_at)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations/2023-09-29-033922_zs_voice_included_samples/down.sql

@@ -0,0 +1,5 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+drop table zs_voice_included_samples;

_sql/migrations/2023-09-29-033922_zs_voice_included_samples/up.sql

@@ -0,0 +1,26 @@
+-- noinspection SqlDialectInspectionForFile
+-- noinspection SqlNoDataSourceInspectionForFile
+-- noinspection SqlResolveForFile
+
+-- In order to rebuild any "zs_voice", we need to know which dataset samples were included.
+-- Since users can change the dataset at any time after a voice was originally created, these
+-- records are generated to retain a record of which samples fed into a given voice.
+-- These records will probably not be used for any product features, just as a means to study
+-- poor-performing voices, perform migrations, etc.
+CREATE TABLE zs_voice_included_samples (
+  -- Not used for anything except replication.
+  id BIGINT(20) NOT NULL AUTO_INCREMENT,
+
+  -- First member of composite effective "primary key"
+  -- The "zs_voice" that was created.
+  voice_token VARCHAR(32) NOT NULL,
+
+  -- Second member of composite effective "primary key"
+  -- The "zs_voice_dataset_sample" that was used.
+  dataset_sample_token VARCHAR(32) NOT NULL,
+
+  -- INDICES --
+  PRIMARY KEY (id),
+  UNIQUE KEY (voice_token, dataset_sample_token)
+
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

_sql/migrations_squashed/tts_tables/tts_models.sql

@@ -83,9 +83,10 @@ CREATE TABLE tts_models (
   -- ========== UNSTRUCTURED METADATA ==========
 
   -- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
+  -- The wide varchar space is because these language tags can become cumbersome, eg "hy-Latn-IT-arevela"
   ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
 
-  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
+  -- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es" and "en-US" becomes "en")
   ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
 
   -- The "name" of the voice model, which might be complicated.