From 53b2aedbb02e715fc8f8c60feffb1d55784e4fc7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 14 Feb 2024 05:27:01 -0500 Subject: [PATCH] Five new schema migrations --- .../down.sql | 6 ++++++ .../up.sql | 6 ++++++ .../down.sql | 5 +++++ .../up.sql | 7 +++++++ .../down.sql | 5 +++++ .../up.sql | 10 ++++++++++ .../down.sql | 5 +++++ .../up.sql | 7 +++++++ .../down.sql | 5 +++++ .../up.sql | 7 +++++++ _sql/migrations_squashed/generic_inference_jobs.sql | 10 ++++++++++ _sql/migrations_squashed/media_files.sql | 8 ++++++++ .../model_weights/model_weights.sql | 5 ++--- .../generic_inference/web/get_inference_job_status.rs | 2 +- 14 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 _sql/migrations/2024-02-14-081740_alter_model_weights_bigint/down.sql create mode 100644 _sql/migrations/2024-02-14-081740_alter_model_weights_bigint/up.sql create mode 100644 _sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/down.sql create mode 100644 _sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/up.sql create mode 100644 _sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/down.sql create mode 100644 _sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/up.sql create mode 100644 _sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/down.sql create mode 100644 _sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/up.sql create mode 100644 _sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/down.sql create mode 100644 _sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/up.sql diff --git a/_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/down.sql b/_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/down.sql new file mode 100644 index 0000000000..29ce8b9275 --- /dev/null +++ b/_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/down.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights + MODIFY COLUMN file_size_bytes INT(10) NOT NULL DEFAULT 0; diff --git a/_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/up.sql b/_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/up.sql new file mode 100644 index 0000000000..6986a8373f --- /dev/null +++ b/_sql/migrations/2024-02-14-081740_alter_model_weights_bigint/up.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE model_weights + MODIFY COLUMN file_size_bytes BIGINT(10) NOT NULL DEFAULT 0; diff --git a/_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/down.sql b/_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/down.sql new file mode 100644 index 0000000000..f6730f7c79 --- /dev/null +++ b/_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE media_files DROP COLUMN maybe_text_transcript; diff --git a/_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/up.sql b/_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/up.sql new file mode 100644 index 0000000000..35397e1c09 --- /dev/null +++ b/_sql/migrations/2024-02-14-085907_alter_media_files_add_text_transcript/up.sql @@ -0,0 +1,7 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE media_files + ADD COLUMN maybe_text_transcript TEXT DEFAULT NULL + AFTER maybe_frame_height; diff --git a/_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/down.sql b/_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/down.sql new file mode 100644 index 0000000000..faeabf90f9 --- /dev/null +++ b/_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE media_files DROP COLUMN maybe_batch_token; diff --git a/_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/up.sql b/_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/up.sql new file mode 100644 index 0000000000..aa0bb6a587 --- /dev/null +++ b/_sql/migrations/2024-02-14-091821_alter_media_files_add_batch_token/up.sql @@ -0,0 +1,10 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE media_files + ADD COLUMN maybe_batch_token VARCHAR(32) DEFAULT NULL + AFTER is_batch_generated; + +ALTER TABLE media_files + ADD INDEX index_maybe_batch_token (maybe_batch_token); diff --git a/_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/down.sql b/_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/down.sql new file mode 100644 index 0000000000..83190ae7f0 --- /dev/null +++ b/_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE generic_inference_jobs DROP COLUMN maybe_download_url; diff --git a/_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/up.sql b/_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/up.sql new file mode 100644 index 0000000000..5bd28105b9 --- /dev/null +++ b/_sql/migrations/2024-02-14-091832_alter_generic_inference_jobs_add_download_url/up.sql @@ -0,0 +1,7 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE generic_inference_jobs + ADD COLUMN maybe_download_url VARCHAR(1024) DEFAULT NULL + AFTER maybe_raw_inference_text; diff --git a/_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/down.sql b/_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/down.sql new file mode 100644 index 0000000000..25a7a80a28 --- /dev/null +++ b/_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE generic_inference_jobs DROP COLUMN on_success_result_batch_token; diff --git a/_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/up.sql b/_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/up.sql new file mode 100644 index 0000000000..1d07d7aeab --- /dev/null +++ b/_sql/migrations/2024-02-14-092306_alter_generic_inference_jobs_add_batch_token/up.sql @@ -0,0 +1,7 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE generic_inference_jobs + ADD COLUMN on_success_result_batch_token VARCHAR(32) DEFAULT NULL + AFTER on_success_result_entity_token; diff --git a/_sql/migrations_squashed/generic_inference_jobs.sql b/_sql/migrations_squashed/generic_inference_jobs.sql index f1fc139c20..40094cb013 100644 --- a/_sql/migrations_squashed/generic_inference_jobs.sql +++ b/_sql/migrations_squashed/generic_inference_jobs.sql @@ -72,6 +72,11 @@ CREATE TABLE generic_inference_jobs ( -- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters. maybe_raw_inference_text TEXT DEFAULT NULL, + -- For download-related jobs, this is the URL + -- This is nullable because not all inference jobs have a download URL. + -- The length of 1024 is arbitrary, but we shouldn't need anything longer. + maybe_download_url VARCHAR(1024) DEFAULT NULL, + -- ========== SUCCESS CASE ========== -- The type of the object will vary based on the type of the upload, @@ -82,6 +87,11 @@ CREATE TABLE generic_inference_jobs ( -- This is only populated on a successful result. on_success_result_entity_token VARCHAR(32) DEFAULT NULL, + -- If the job completes successfully, this is the batch token (ie. when + -- several results are created). + -- Only populated if there are actual batches generated. + on_success_result_batch_token VARCHAR(32) DEFAULT NULL, + -- ========== CREATOR DETAILS AND PREFERENCES ========== -- Foreign key to user diff --git a/_sql/migrations_squashed/media_files.sql b/_sql/migrations_squashed/media_files.sql index 35d7334ca9..fce17613f1 100644 --- a/_sql/migrations_squashed/media_files.sql +++ b/_sql/migrations_squashed/media_files.sql @@ -58,11 +58,15 @@ CREATE TABLE media_files ( -- The original filename of the media (if uploaded by a user) maybe_origin_filename VARCHAR(255) DEFAULT NULL, + -- TODO: Remove after `maybe_batch_token` gains use. -- Whether this media file was generated as part of a batch -- If so, we can look up the batch in a separate query/call to the `batch_generations` table. -- We won't hold the batch token in this table since it'll be very sparsely populated. is_batch_generated BOOLEAN NOT NULL DEFAULT FALSE, + -- If the media file is generated as part of a batch, this designates the batch. + maybe_batch_token VARCHAR(32) DEFAULT NULL, + -- ========== METADATA ========== -- We can store text transcripts, etc. in other tables and refer back via foreign key. @@ -100,6 +104,9 @@ CREATE TABLE media_files ( -- For videos, the original frame height. maybe_frame_height INT(5) DEFAULT NULL, + -- Optional text transcript for audio or video (especially TTS) + maybe_text_transcript TEXT DEFAULT NULL, + -- Checksum of the original media -- SHA1 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)] -- checksum_sha1 CHAR(40) NOT NULL, @@ -212,6 +219,7 @@ CREATE TABLE media_files ( KEY index_maybe_origin_model_type (maybe_origin_model_type), KEY fk_maybe_origin_model_token (maybe_origin_model_token), KEY fk_maybe_origin_model_type_and_token (maybe_origin_model_type, maybe_origin_model_token), + KEY index_maybe_batch_token (maybe_batch_token), KEY index_media_type (media_type), KEY index_checksum_sha2 (checksum_sha2), KEY fk_maybe_creator_user_token (maybe_creator_user_token), diff --git a/_sql/migrations_squashed/model_weights/model_weights.sql b/_sql/migrations_squashed/model_weights/model_weights.sql index 2cfa5681d1..9e6125ec1b 100644 --- a/_sql/migrations_squashed/model_weights/model_weights.sql +++ b/_sql/migrations_squashed/model_weights/model_weights.sql @@ -95,9 +95,8 @@ CREATE TABLE model_weights ( -- The filename that was used at upload time (if available) original_filename VARCHAR(255) DEFAULT NULL, - -- TODO: Make this a BIGINT so we can store more than 2GB. - -- File characteristics - file_size_bytes INT(10) NOT NULL DEFAULT 0, + -- File size. BIGINT to store more than 2GB size. + file_size_bytes BIGINT(10) NOT NULL DEFAULT 0, -- Checksum of the original weights file -- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)] diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 68c49915e5..771d992aac 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -181,7 +181,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI if let Some(title) = record.maybe_model_weights_title.as_deref() { maybe_model_title = Some(title); } - + if maybe_model_title.is_none() { maybe_model_title = match record.inference_category { InferenceCategory::LipsyncAnimation => Some("lipsync animation"),