daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-05 09:48:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23 from storytold/brandon-web-vc

Browse Source 
Web-based Voice Conversion Part 1
This commit is contained in:
Brandon Thomas authored and GitHub committed 2023-01-11 03:06:15 -05:00
2 parents 587ea70d33 30202e6e5a
commit bc033c2202
476 files changed
+7321 -1046
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.env-download-job
+8
View File
@@ -6,11 +6,19 @@ WEB_DOWNLOADER_COMMAND=./download_internet_file.py
WEB_DOWNLOADER_MAYBE_VENV_SCRIPT=./python/bin/activate
WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d73f28ce3ff6
SOFTVC_MODEL_CHECK_ROOT_DIRECTORY=/
SOFTVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
TACOTRON_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
TACOTRON_MODEL_CHECK_COMMAND=./vocodes_model_check_tacotron.py
TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
HIFIGAN_SOFTVC_MODEL_CHECK_ROOT_DIRECTORY=/
HIFIGAN_SOFTVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
HIFIGAN_SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3
HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
HIFIGAN_MODEL_CHECK_COMMAND=./vocodes_model_check_hifigan.py
.idea/dictionaries/bt.xml
Generated
+5
View File
@@ -7,9 +7,14 @@
<w>cheermote</w>
<w>crockford</w>
<w>fakeyou</w>
<w>ffprobe</w>
<w>hifigan</w>
<w>scrollback</w>
<w>softvc</w>
<w>symphonia</w>
<w>tacotron</w>
<w>transcoded</w>
<w>transcodings</w>
<w>upserting</w>
<w>vocoder</w>
<w>vocoders</w>
Cargo.lock
Generated
+314 -37
View File
@@ -667,6 +667,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "arrayvec"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "ascii"
version = "0.9.3"
@@ -711,7 +717,7 @@ dependencies = [
"smol_str",
"thiserror",
"tokio 1.16.1",
"uuid",
"uuid 0.8.2",
]
[[package]]
@@ -975,12 +981,26 @@ dependencies = [
"libc",
]
[[package]]
name = "buckets"
version = "0.0.1"
dependencies = [
"crockford",
"errors",
]
[[package]]
name = "bumpalo"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
[[package]]
name = "bytemuck"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f"
[[package]]
name = "byteorder"
version = "1.4.3"
@@ -1040,7 +1060,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca453e8624711b2f0f4eb47076a318feda166252a827ee25d067b43de83dcba0"
dependencies = [
"byteorder",
"uuid",
"uuid 0.8.2",
]
[[package]]
name = "cfb"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"
dependencies = [
"byteorder",
"fnv",
"uuid 1.2.2",
]
[[package]]
@@ -1115,6 +1146,18 @@ dependencies = [
"os_str_bytes",
]
[[package]]
name = "cloud_storage"
version = "0.0.1"
dependencies = [
"anyhow",
"buckets",
"container_common",
"log",
"rust-s3",
"tokio 1.16.1",
]
[[package]]
name = "cloudabi"
version = "0.0.3"
@@ -1205,12 +1248,12 @@ dependencies = [
"log",
"once_cell",
"rand 0.8.4",
"ring 0.17.0-alpha.10",
"ring 0.17.0-alpha.11",
"serde",
"serde_json",
"tempfile",
"toml",
"uuid",
"uuid 0.8.2",
]
[[package]]
@@ -1314,6 +1357,13 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crockford"
version = "0.0.1"
dependencies = [
"rand 0.8.4",
]
[[package]]
name = "crossbeam"
version = "0.8.1"
@@ -1415,9 +1465,9 @@ dependencies = [
[[package]]
name = "data-encoding"
version = "2.3.2"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb"
[[package]]
name = "database-backup"
@@ -1462,10 +1512,13 @@ version = "0.0.1"
dependencies = [
"anyhow",
"base64 0.13.0",
"buckets",
"chrono",
"config",
"container_common",
"dotenv",
"enums",
"errors",
"log",
"once_cell",
"regex",
@@ -1618,11 +1671,13 @@ dependencies = [
"anyhow",
"chrono",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
"database_queries",
"dotenv",
"easyenv",
"enums",
"google_drive_common",
"hostname",
"hyper 0.14.14",
@@ -1636,7 +1691,6 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"subprocess_common",
"tempdir",
@@ -1691,6 +1745,19 @@ dependencies = [
"syn",
]
[[package]]
name = "enums"
version = "0.0.1"
dependencies = [
"serde",
"serde_derive",
"serde_json",
"sqlx",
"sqlx-core",
"strum",
"toml",
]
[[package]]
name = "env_logger"
version = "0.7.1"
@@ -1704,6 +1771,13 @@ dependencies = [
"termcolor",
]
[[package]]
name = "errors"
version = "0.0.1"
dependencies = [
"anyhow",
]
[[package]]
name = "event-listener"
version = "2.5.3"
@@ -1719,6 +1793,15 @@ dependencies = [
"instant",
]
[[package]]
name = "files"
version = "0.0.1"
dependencies = [
"data-encoding",
"infer 0.11.0",
"ring 0.17.0-alpha.11",
]
[[package]]
name = "firestorm"
version = "0.4.6"
@@ -2353,6 +2436,7 @@ dependencies = [
"base64 0.13.0",
"bcrypt",
"chrono",
"cloud_storage",
"config",
"container_common",
"data-encoding",
@@ -2392,14 +2476,13 @@ dependencies = [
"regex",
"reqwest",
"reusable_types",
"ring 0.17.0-alpha.10",
"ring 0.17.0-alpha.11",
"rust-s3",
"serde",
"serde_derive",
"serde_json",
"sha2 0.9.5",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"time 0.2.26",
@@ -2408,7 +2491,7 @@ dependencies = [
"tts_common",
"twitch_common",
"user_input_common",
"uuid",
"uuid 0.8.2",
]
[[package]]
@@ -2433,7 +2516,16 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea70330449622910e0edebab230734569516269fb32342fb0a8956340fa48c6c"
dependencies = [
"cfb",
"cfb 0.4.0",
]
[[package]]
name = "infer"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a6c16b11a665b26aeeb9b1d7f954cdeb034be38dd00adab4f2ae921a8fee804"
dependencies = [
"cfb 0.7.3",
]
[[package]]
@@ -2443,11 +2535,14 @@ dependencies = [
"anyhow",
"chrono",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
"database_queries",
"dotenv",
"easyenv",
"enums",
"errors",
"hostname",
"hyper 0.14.14",
"jobs_common",
@@ -2460,9 +2555,9 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"tokens",
"tokio 1.16.1",
"tts_common",
]
@@ -2607,7 +2702,7 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
dependencies = [
"arrayvec",
"arrayvec 0.5.2",
"bitflags",
"cfg-if 1.0.0",
"ryu",
@@ -2800,6 +2895,15 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "media"
version = "0.0.1"
dependencies = [
"errors",
"log",
"symphonia",
]
[[package]]
name = "memchr"
version = "2.4.0"
@@ -3007,7 +3111,7 @@ dependencies = [
"serde",
"serde_json",
"tokio 1.16.1",
"uuid",
"uuid 0.8.2",
]
[[package]]
@@ -3827,9 +3931,9 @@ dependencies = [
[[package]]
name = "ring"
version = "0.17.0-alpha.10"
version = "0.17.0-alpha.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55ae9a4d2975bdd8254d7bcdd2261be62b42b8920f903ec682b08cdc11b87af2"
checksum = "4575a179070909595bea5f999d67934737c2e0757a1eb9839af555917817b257"
dependencies = [
"cc",
"libc",
@@ -4475,17 +4579,6 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
[[package]]
name = "storage_buckets_common"
version = "0.0.1"
dependencies = [
"anyhow",
"container_common",
"log",
"rust-s3",
"tokio 1.16.1",
]
[[package]]
name = "storyteller-web"
version = "0.0.1"
@@ -4505,7 +4598,9 @@ dependencies = [
"base64 0.13.0",
"bcrypt",
"billing_component",
"buckets",
"chrono",
"cloud_storage",
"config",
"container_common",
"data-encoding",
@@ -4514,6 +4609,8 @@ dependencies = [
"diesel",
"dotenv",
"easyenv",
"enums",
"files",
"futures 0.1.31",
"futures 0.3.17",
"futures-util",
@@ -4535,6 +4632,7 @@ dependencies = [
"lru_time_cache",
"magic-crypt",
"md-5",
"media",
"memory_caching",
"native-tls",
"newrelic-telemetry",
@@ -4546,7 +4644,7 @@ dependencies = [
"regex",
"reqwest",
"reusable_types",
"ring 0.17.0-alpha.10",
"ring 0.17.0-alpha.11",
"rust-s3",
"serde",
"serde_derive",
@@ -4554,8 +4652,8 @@ dependencies = [
"sha2 0.9.5",
"speculoos",
"sqlx",
"storage_buckets_common",
"subprocess",
"symphonia",
"tempdir",
"time 0.2.26",
"tokens",
@@ -4568,7 +4666,7 @@ dependencies = [
"url_config",
"user_input_common",
"users_component",
"uuid",
"uuid 0.8.2",
]
[[package]]
@@ -4639,6 +4737,178 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "symphonia"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17033fe05e4f7f10a6ad602c272bafd2520b2e5cdd9feb61494d9cdce08e002f"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-alac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-isomp4",
"symphonia-format-mkv",
"symphonia-format-ogg",
"symphonia-format-wav",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "044f655337892b217d6df1d8336ee119414c3886c89c72e0156989cd2ad7934a"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db5d3d53535ae2b7d0e39e82f683cac5398a6c8baca25ff1183e107d13959d3e"
dependencies = [
"bitflags",
"lazy_static",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e9431b89428c31b01428563df18e52b1aff7c49e71fb80b2fa8e85632094776"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-alac"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "452438d6f32bf07f2f55f35371c3c8e9cce4a028a08b47fb301001f85a950f02"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17cca412c954abda6ab62b5e51223568eb604ed8266ec777d99e1d63c608443c"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "323b94435a1a807e1001e29490aeaef2660fb72b145d47497e8429a6cb1d67c3"
dependencies = [
"log",
"symphonia-core",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "199a6417cd4115bac79289b64b859358ea050b7add0ceb364dc991f628c5b347"
dependencies = [
"arrayvec 0.7.2",
"bitflags",
"bytemuck",
"lazy_static",
"log",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dce84cea63247dfcc43e12edf2d19406fbb62ebc24bac1b8e6c49f1ba67b890"
dependencies = [
"encoding_rs",
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-mkv"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f405400330b2cc0c70e19515198628ae9a6a99a59c77800541127d4cff113b96"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d2f741469a0f103607ed1f2605f7f00b13ba044ea9ddc616764558c6d3d9b7d"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-wav"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9d771aa6889f05b771e629110f6a60b5e1c0ad580fc41da574bc490fbe2822"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed71acf6b5e6e8bee1509597b86365a06b78c1d73218df47357620a6fe5997b"
dependencies = [
"encoding_rs",
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-utils-xiph"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbb0766ce77a8aef535f9438db645e7b6f1b2c4cf3be9bf246b4e11a7d5531"
dependencies = [
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "1.0.103"
@@ -4843,7 +5113,6 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
name = "tokens"
version = "0.0.1"
dependencies = [
"anyhow",
"rand 0.8.4",
"serde",
"serde_derive",
@@ -5173,6 +5442,7 @@ dependencies = [
"anyhow",
"chrono",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
"database_queries",
@@ -5190,7 +5460,6 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"tokio 1.16.1",
@@ -5204,6 +5473,7 @@ dependencies = [
"anyhow",
"chrono",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
"database_queries",
@@ -5221,7 +5491,6 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"tokio 1.16.1",
@@ -5329,7 +5598,7 @@ dependencies = [
"twitch_api2",
"twitch_common",
"twitch_oauth2",
"uuid",
"uuid 0.8.2",
]
[[package]]
@@ -5555,6 +5824,7 @@ dependencies = [
"container_common",
"database_queries",
"dotenv",
"enums",
"hmac 0.10.1",
"http_server_common",
"jwt",
@@ -5590,6 +5860,12 @@ dependencies = [
"getrandom 0.2.2",
]
[[package]]
name = "uuid"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
[[package]]
name = "vcpkg"
version = "0.2.12"
@@ -5619,8 +5895,10 @@ name = "w2l-download-job"
version = "0.0.1"
dependencies = [
"anyhow",
"buckets",
"chrono",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
"database_queries",
@@ -5638,7 +5916,6 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"tokio 1.16.1",
@@ -5652,6 +5929,7 @@ dependencies = [
"anyhow",
"chrono",
"clap 2.34.0",
"cloud_storage",
"config",
"container_common",
"database_queries",
@@ -5669,7 +5947,6 @@ dependencies = [
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"tokio 1.16.1",
@@ -5862,7 +6139,7 @@ dependencies = [
"twitch_api2",
"twitch_common",
"twitch_oauth2",
"uuid",
"uuid 0.8.2",
]
[[package]]
Cargo.toml
+22 -13
View File
@@ -1,6 +1,13 @@
[workspace]
members = [
# Types: common strongly-typed tokens, ids, payloads, etc.
# Also common cloud buckets and other "strongly typed" configs.
"crates/_types/buckets",
"crates/_types/enums",
"crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
"crates/_types/tokens",
# CLI tools for operations, etc.
"crates/cli/database_backup",
"crates/cli/database_cleanup",
@@ -11,32 +18,34 @@ members = [
"crates/plugins/users",
# Shared libraries
"crates/lib/cloud_storage",
"crates/lib/config",
"crates/lib/container_common",
"crates/lib/crockford",
"crates/lib/database_queries",
"crates/lib/errors",
"crates/lib/files",
"crates/lib/google_drive_common",
"crates/lib/http_server_common",
"crates/lib/jobs_common",
"crates/lib/media",
"crates/lib/memory_caching",
"crates/lib/redis_common",
"crates/lib/reusable_types",
"crates/lib/storage_buckets_common",
"crates/lib/subprocess_common",
"crates/lib/tokens",
"crates/lib/tts_common",
"crates/lib/twitch_common",
"crates/lib/url_config",
"crates/lib/user_input_common",
# Services: API servers, jobs, workers, etc.
"crates/service/download_job",
"crates/service/image_generation_service",
"crates/service/inference_job",
"crates/service/storyteller_web",
"crates/service/tts_download_job",
"crates/service/tts_inference_job",
"crates/service/twitch_pubsub_subscriber",
"crates/service/w2l_download_job",
"crates/service/w2l_inference_job",
"crates/service/websocket_gateway",
"crates/service/job/download_job",
"crates/service/job/inference_job",
"crates/service/job/tts_download_job",
"crates/service/job/tts_inference_job",
"crates/service/job/w2l_download_job",
"crates/service/job/w2l_inference_job",
"crates/service/other/twitch_pubsub_subscriber",
"crates/service/web/image_generation_service",
"crates/service/web/storyteller_web",
"crates/service/web/websocket_gateway",
]
Dockerfile
+2 -1
View File
@@ -9,7 +9,7 @@
FROM ubuntu:jammy as rust-base
# NB: This can be "stable" or another version.
ARG RUST_TOOLCHAIN="1.65.0"
ARG RUST_TOOLCHAIN="1.66.0"
WORKDIR /tmp
@@ -64,6 +64,7 @@ COPY Cargo.toml .
COPY sqlx-data.json .
COPY crates/ ./crates
COPY db/ ./db
COPY test_data/ ./test_data
# Run all of the tests
RUN SQLX_OFFLINE=true \
_migrations/2022-08-05-202017_generic_download_jobs/up.sql
+2 -1
View File
@@ -22,12 +22,13 @@ CREATE TABLE generic_download_jobs (
-- The type of the object will vary based on the type of the upload,
-- and we may include heuristics that auto-detect types in the future
-- Example types: 'hifigan', 'hifigan_rocket_vc' (codename for softvc), 'tacotron2', ...
on_success_downloaded_entity_type VARCHAR(32) DEFAULT NULL,
-- ========== UPLOAD DETAILS ==========
-- A "well defined" type of upload
-- The first upload type will be "hifigan_model"
-- Examples: "hifigan", "tacotron2", etc.
download_type VARCHAR(32) NOT NULL,
-- Assume the same URL can be uploaded more than once (in case of errors or
_migrations/2022-08-05-202110_vocoder_models/up.sql
+3 -1
View File
@@ -10,7 +10,9 @@ CREATE TABLE vocoder_models (
token VARCHAR(32) NOT NULL,
-- The architecture of the vocoder.
-- Currently supported values: 'hifigan'
-- Currently supported values:
-- * 'hifigan' - hifigan for Tacotron2
-- * 'hifigan_rocket_vc' - hifigan for SoftVC ("rocket" is a codename)
vocoder_type VARCHAR(32) NOT NULL,
-- ========== MUTABLE METADATA ==========
_migrations/2022-12-14-050849_generic_inference_jobs/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
DROP TABLE generic_inference_jobs;
_migrations/2022-12-14-050849_generic_inference_jobs/up.sql
+150
View File
@@ -0,0 +1,150 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
CREATE TABLE generic_inference_jobs (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" (PUBLIC)
-- This is so the in-progress results can be looked up by the UI.
token VARCHAR(32) NOT NULL,
-- Idempotency token from client
-- This is so the frontend client doesn't submit duplicate jobs.
uuid_idempotency_token VARCHAR(36) NOT NULL,
-- ========== INFERENCE DETAILS ==========
-- Type of inference
inference_type VARCHAR(32) NOT NULL,
-- Polymorphic arguments payload that depends on the type of inference job.
-- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
maybe_inference_args TEXT DEFAULT NULL,
-- For text-based workloads, the raw, unprocessed user input.
-- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
maybe_raw_inference_text TEXT DEFAULT NULL,
-- Potential foreign key to the primary model being used, if any.
-- This will normally live in `maybe_inference_args`, but in this case, it's useful for
-- running easy database analytical queries.
maybe_model_token VARCHAR(32) DEFAULT NULL,
-- ========== SUCCESS CASE ==========
-- The type of the object will vary based on the type of the upload,
-- and we may include heuristics that auto-detect types in the future
on_success_result_entity_type VARCHAR(32) DEFAULT NULL,
-- If the job completes successfully, this is the result token.
-- This is only populated on a successful result.
on_success_result_entity_token VARCHAR(32) DEFAULT NULL,
-- ========== CREATOR DETAILS AND PREFERENCES ==========
-- Foreign key to user
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
-- Based on a cookie sent by the frontend.
-- We'll save this even if the user is logged in.
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-- For abuse tracking.
-- Wide enough for IPv4/6
creator_ip_address VARCHAR(40) NOT NULL,
-- The creator can set a desired visibility for their data.
-- This does not always apply to every upload type.
-- Additionally, some upload types may require moderator approval prior
-- to being publicly listed, and this field has no bearing on that.
-- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
creator_set_visibility ENUM(
'public',
'hidden',
'private'
) NOT NULL DEFAULT 'public',
-- ========== PREMIUM FEATURES METADATA ==========
-- The maximum duration for generated audio in seconds.
-- Zero is implied to be the default value, which is typically 12 seconds.
-- A negative value implies "unlimited".
max_duration_seconds INTEGER NOT NULL DEFAULT 0,
-- ========== WORKLOAD SOURCE AND PRIORITY ==========
-- Priority *increases*, so a level of 2 will be higher than 1.
-- These are the level semantics currently:
-- - All jobs from anonymous FakeYou users have level 0.
-- - All jobs from logged in FakeYou users have level 1.
-- - All jobs from Twitch TTS (unpaid) have level 10 (ten).
-- - (There will be future levels for paid Twitch and social FakeYou rewards.)
priority_level SMALLINT UNSIGNED NOT NULL DEFAULT 0,
is_from_premium_user BOOLEAN NOT NULL DEFAULT FALSE,
is_from_api_user BOOLEAN NOT NULL DEFAULT FALSE,
is_for_twitch BOOLEAN NOT NULL DEFAULT FALSE,
-- If true, the request gets routed to a special "debug" worker.
is_debug_request BOOLEAN NOT NULL DEFAULT FALSE,
-- ========== JOB SYSTEM DETAILS ==========
-- Jobs begin as "pending", then transition to other states.
--
-- * Pending = job is ready to go
-- * Started = job is running
-- * Complete_Success = job is done (success)
-- * Complete_Failure = job is done (failure)
-- * Attempt_Failed = job failed but may retry.
-- * Dead = job failed permanently.
--
-- Pending -> Started -> Complete_Success
-- |-> Complete_Failure
-- \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
status ENUM(
'pending',
'started',
'complete_success',
'complete_failure',
'attempt_failed',
'dead') NOT NULL DEFAULT 'pending',
-- We can track this against a "max_attempt_count"
attempt_count SMALLINT UNSIGNED NOT NULL DEFAULT 0,
-- If there is a failure, tell the user why.
failure_reason VARCHAR(512) DEFAULT NULL,
-- Optional internal-only debugging information in the case of failure.
internal_debugging_failure_reason VARCHAR(512) DEFAULT NULL,
-- The last worker (hostname or pod name) to touch the job, either in the case of success or failure.
last_assigned_worker VARCHAR(255) DEFAULT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- Failed jobs will set a next attempt time.
-- Subsequent tries can increase the timeout.
-- Failures because of permissions require human intervention => [retry_at=null].
-- Failures because of invalid files are dead => [status=dead].
retry_at TIMESTAMP NULL,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (token),
UNIQUE KEY (uuid_idempotency_token),
KEY index_inference_type (inference_type),
KEY fk_maybe_model_token (maybe_model_token),
KEY fk_on_success_result_entity_token (on_success_result_entity_token),
KEY fk_maybe_creator_user_token (maybe_creator_user_token),
KEY index_creator_ip_address (creator_ip_address),
KEY index_priority_level (priority_level),
KEY index_is_debug_request (is_debug_request),
KEY index_status (status)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
_migrations/2022-12-15-060826_media_uploads/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
drop table media_uploads;
_migrations/2022-12-15-060826_media_uploads/up.sql
+135
View File
@@ -0,0 +1,135 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
CREATE TABLE media_uploads (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" (PUBLIC)
-- This is so the in-progress results can be looked up by the UI.
token VARCHAR(32) NOT NULL,
-- Idempotency token from client
-- This is so the frontend client doesn't submit duplicate jobs.
uuid_idempotency_token VARCHAR(36) NOT NULL,
-- ========== MEDIA DETAILS ==========
-- Type of media:
-- * 'audio' for wav, mp3, etc.
-- * 'video' for a variety of video types.
media_type VARCHAR(16) NOT NULL,
-- The original filename of the media
maybe_original_filename VARCHAR(255) DEFAULT NULL,
-- The original (non-transcoded or truncated) file size
original_file_size_bytes INT(10) NOT NULL DEFAULT 0,
-- The original (non-transcoded or truncated) duration in millis.
original_duration_millis INT(10) NOT NULL DEFAULT 0,
-- The original file's mime type.
maybe_original_mime_type VARCHAR(32) DEFAULT NULL,
-- Audio encoder details
-- Only present if the file was audio or a video that had audio.
maybe_original_audio_encoding VARCHAR(32) DEFAULT NULL,
-- Video encoder details
-- Only present if the file was a video.
maybe_original_video_encoding VARCHAR(32) DEFAULT NULL,
-- For videos, the original frame width.
maybe_original_frame_width INT(5) DEFAULT NULL,
-- For videos, the original frame height.
maybe_original_frame_height INT(5) DEFAULT NULL,
-- Checksum of the original media
-- SHA1 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
-- checksum_sha1 CHAR(40) NOT NULL,
-- Checksum of the original media
-- SHA2 hash [SHA2 = CHAR(64), SHA1 = CHAR(40), MD5 = CHAR(32)]
-- Note that SHA2 is a hash family (SHA-228, SHA-256, SHA-384, SHA-512, SHA-512/224, SHA-512/256, ...)
-- and produces different lengths output depending on the choice of algorithm.
checksum_sha2 CHAR(64) NOT NULL,
-- ========== UPLOAD, TRANSCODING, AND TRUNCATION DETAILS ==========
-- The hash for the bucket directory that contains the original upload
-- as well as any transcodings, downsamplings, etc.
public_bucket_directory_hash VARCHAR(32) NOT NULL,
-- TODO(bt, 2022-12-20): Before landing, rename this field be better and indicate this
-- is a *directory*. Add a second field to contain the default file name.
-- The directory this media is uploaded to will be exclusive for this file.
-- Only this given record will live in this bucket, but the directory may include
-- other transcodings or truncations.
public_bucket_directory_full_path VARCHAR(255) NOT NULL,
-- We'll likely transcode (and potentially truncate) most media given to us.
-- This will store a json-encoded struct that details the changes.
-- TEXT = 65,535 bytes (64 KiB), ~= 4 bytes per UTF-8 character, ~ 16383 characters.
extra_file_modification_info TEXT DEFAULT NULL,
-- ========== CREATOR DETAILS AND PREFERENCES ==========
-- Foreign key to user
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
-- Based on a cookie sent by the frontend.
-- We'll save this even if the user is logged in.
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-- For abuse tracking.
-- Wide enough for IPv4/6
creator_ip_address VARCHAR(40) NOT NULL,
-- The creator can set a desired visibility for their data.
-- This does not always apply to every upload type.
-- Additionally, some upload types may require moderator approval prior
-- to being publicly listed, and this field has no bearing on that.
-- NB: DO NOT CHANGE THE ORDER OF THE ENUM VALUES DURING SCHEMA MIGRATIONS.
creator_set_visibility ENUM(
'public',
'hidden',
'private'
) NOT NULL DEFAULT 'public',
-- The synthetic id associated with this result.
-- These ids are incremented on a per-user basis to help users
-- sequence their own work. They serve no other purpose.
maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL,
-- ========== MODERATION DETAILS ==========
-- The last moderator that made changes last.
-- We should also build an audit table to supersede this.
maybe_mod_user_token VARCHAR(32) DEFAULT NULL,
-- ========== TIMESTAMPS ==========
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- If this is removed by a mod or the creator.
user_deleted_at TIMESTAMP NULL,
mod_deleted_at TIMESTAMP NULL,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (token),
UNIQUE KEY (uuid_idempotency_token),
KEY index_media_type (media_type),
KEY index_checksum_sha2 (checksum_sha2),
KEY fk_maybe_creator_user_token (maybe_creator_user_token),
KEY fk_maybe_mod_user_token (maybe_mod_user_token),
KEY index_creator_ip_address (creator_ip_address),
KEY index_user_deleted_at (user_deleted_at),
KEY index_mod_deleted_at (mod_deleted_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
_migrations/2022-12-15-060830_audit_logs/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
drop table audit_logs;
_migrations/2022-12-15-060830_audit_logs/up.sql
+48
View File
@@ -0,0 +1,48 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
CREATE TABLE audit_logs (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" of the audit log item (in case we add a UI or notes)
token VARCHAR(32) NOT NULL,
-- The type of entity mutated
entity_type VARCHAR(32) NOT NULL,
-- The token of the entity mutated
entity_token VARCHAR(32) NOT NULL,
-- The type of action taken on the entity
entity_action VARCHAR(32) NOT NULL,
-- Typically only signed in users will be performing actions on known entities,
-- but we're leaving this null just in case. This is the corresponding user that
-- made the change.
maybe_actor_user_token VARCHAR(32) DEFAULT NULL,
-- Based on a cookie sent by the frontend.
-- We'll save this even if the user is logged in.
maybe_actor_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-- Whether the actor has some sort of moderator privilege.
is_actor_moderator BOOLEAN NOT NULL DEFAULT FALSE,
-- For abuse tracking.
-- Wide enough for IPv4/6
actor_ip_address VARCHAR(40) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (token),
KEY fk_entity_type (entity_type),
KEY fk_entity_token (entity_token),
KEY fk_maybe_actor_user_token (maybe_actor_user_token)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
_migrations/2022-12-15-223359_voice_conversion_models/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
DROP TABLE voice_conversion_models;
_migrations/2022-12-15-223359_voice_conversion_models/up.sql
+144
View File
@@ -0,0 +1,144 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
CREATE TABLE voice_conversion_models (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" (PUBLIC)
token VARCHAR(32) NOT NULL,
-- ========== MODEL FUNCTIONALITY ==========
-- The type of voice conversion model
-- eg. 'rocket_vc' (codename for 'softvc'), 'diffsvc'
model_type VARCHAR(32) NOT NULL,
-- If set, we'll use the vocoder defined.
-- If not set, some global default will be used.
maybe_vocoder_token VARCHAR(32) DEFAULT NULL,
-- ========== MODEL DESCRIPTION ==========
-- The "name" of the voice model, which might be complicated.
-- If maybe_subject_token (etc.) is set, then it's authoritative instead.
title VARCHAR(255) NOT NULL,
-- The description of the model in markdown.
description_markdown TEXT NOT NULL,
-- Generated HTML (not user-editable).
description_rendered_html TEXT NOT NULL,
-- The full IETF BCP47 language tag (eg. en, en-US, es-419, ja-JP, pt, etc.)
-- (Not that it matters apart from categorization, since voice conversion is universal.)
ietf_language_tag VARCHAR(64) NOT NULL DEFAULT 'en',
-- The IETF BCP47 language tag's primary language subtag (eg. "es-419" becomes "es")
-- (Not that it matters apart from categorization, since voice conversion is universal.)
ietf_primary_language_subtag VARCHAR(12) NOT NULL DEFAULT 'en',
-- ========== MODEL PLACEMENT ==========
-- Whether the voice features on the front of FakeYou
is_front_page_featured BOOLEAN NOT NULL DEFAULT FALSE,
-- ========== IMMUTABLE PROVENANCE AND METADATA ==========
-- Where the file was originally downloaded (if it was downloaded)
original_download_url VARCHAR(512) DEFAULT NULL,
-- The filename that was used at upload time (if available)
original_filename VARCHAR(255) DEFAULT NULL,
-- File characteristics
file_size_bytes INT(10) NOT NULL DEFAULT 0,
-- ========== CREATOR DETAILS ==========
-- The person that created the template.
creator_user_token VARCHAR(32) NOT NULL,
-- For abuse tracking.
-- Wide enough for IPv4/6.
-- This will pick up moderator IP addresses, but a future audit logs table
-- will capture all edits.
ip_address_creation VARCHAR(40) NOT NULL,
ip_address_last_update VARCHAR(40) NOT NULL,
-- The last user to edit the model. This could be the creator or a moderator.
-- Check the future audit logs table for more info.
maybe_last_update_user_token VARCHAR(32) DEFAULT NULL,
-- (THIS MIGHT NOT BE USED)
-- NB: DO NOT SORT!
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
creator_set_visibility ENUM(
'public',
'hidden',
'private'
) NOT NULL DEFAULT 'public',
-- ========== BUCKET STORAGE ==========
-- The pytorch model
-- For now, this will be a hash of the file contents.
-- NB: NOT UNIQUE! We can allow duplicate uploads.
private_bucket_hash CHAR(64) NOT NULL,
-- The "full url" version of the path
private_bucket_object_name VARCHAR(255) NOT NULL,
-- ========== RATING AND RANKING ==========
-- Calculated average, on a scale of 0-100
-- Null with zero ratings.
-- calculated_average_score INT(3) DEFAULT NULL,
-- Other metrics
-- calculated_total_ratings_submitted_count INT(10) NOT NULL DEFAULT 0,
-- calculated_total_uses_count BIGINT(10) NOT NULL DEFAULT 0,
-- ========== MODERATION DETAILS ==========
-- Mods may have to approve of vc models for them to show up in a public index.
-- Models can be used by the author (or other parties that know the URL) before
-- they're approved, but unapproved models won't show up in public indices.
-- This is independent of other visibility controls (is_mod_disabled, deleted_at,
-- and creator_set_visibility)
is_public_listing_approved BOOLEAN DEFAULT NULL,
-- Extremely popular models may be locked from deletion or modification by users.
is_locked_from_user_modification BOOLEAN NOT NULL DEFAULT FALSE,
-- If a moderator has comments.
maybe_mod_comments VARCHAR(255) DEFAULT NULL,
-- ========== VECTOR CLOCK ==========
-- Incremented with every update.
version INT NOT NULL DEFAULT 0,
-- ========== TIMESTAMPS ==========
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- If this is removed by a mod.
-- It completely disappears from the system.
user_deleted_at TIMESTAMP NULL,
mod_deleted_at TIMESTAMP NULL,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (token),
KEY index_model_type (model_type),
KEY fk_maybe_vocoder_token (maybe_vocoder_token),
KEY fk_creator_user_token (creator_user_token),
KEY index_creator_set_visibility (creator_set_visibility),
KEY index_private_bucket_hash (private_bucket_hash),
KEY index_is_public_listing_approved (is_public_listing_approved),
KEY index_is_locked_from_user_modification (is_locked_from_user_modification)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
_migrations/2022-12-15-223422_voice_conversion_results/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
DROP TABLE voice_conversion_results;
_migrations/2022-12-15-223422_voice_conversion_results/up.sql
+106
View File
@@ -0,0 +1,106 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
CREATE TABLE voice_conversion_results (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" (PUBLIC)
token VARCHAR(32) NOT NULL,
-- ========== INFERENCE DETAILS ==========
-- The TTS model that was used
model_token VARCHAR(32) NOT NULL,
-- If set, specifies which vocoder was used.
maybe_vocoder_token VARCHAR(64) DEFAULT NULL,
-- Which media token was used (can be audio or video)
media_token VARCHAR(32) NOT NULL,
-- ========== CREATOR DETAILS ==========
-- The person that created the result
-- If the user wasn't logged in, this is null
maybe_creator_user_token VARCHAR(32) DEFAULT NULL,
-- Based on a cookie sent by the frontend.
-- We'll save this even if the user is logged in.
maybe_creator_anonymous_visitor_token VARCHAR(32) DEFAULT NULL,
-- For abuse tracking.
-- Wide enough for IPv4/6
creator_ip_address VARCHAR(40) NOT NULL,
-- (THIS MIGHT NOT BE USED)
-- NB: DO NOT SORT!
-- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
creator_set_visibility ENUM(
'public',
'hidden',
'private'
) NOT NULL DEFAULT 'public',
-- The synthetic id associated with this result.
-- These ids are incremented on a per-user basis to help users
-- sequence their own work. They serve no other purpose.
maybe_creator_synthetic_id BIGINT(20) DEFAULT NULL,
-- ========== BUCKET STORAGE ==========
-- Full path where the wav, spectrogram, video, etc. are located.
public_bucket_directory_full_path VARCHAR(255) NOT NULL,
-- Name of the wav audio file (if it exists)
public_bucket_wav_audio_object_name VARCHAR(255) NOT NULL,
-- Name of the spectrogram (if it exists)
public_bucket_spectrogram_object_name VARCHAR(255) NOT NULL,
-- Name of the video (if it exists)
public_bucket_video_object_name VARCHAR(255) NOT NULL,
-- ========== METADATA ==========
file_size_bytes INT(10) NOT NULL DEFAULT 0,
duration_millis INT(10) NOT NULL DEFAULT 0,
-- ========== INFERENCE METADATA, DC, ROUTING, DEBUGGING ==========
-- If true, the request was routed to a special "debug" worker.
is_debug_request BOOLEAN NOT NULL DEFAULT FALSE,
is_generated_on_prem BOOLEAN NOT NULL DEFAULT FALSE,
generated_by_worker VARCHAR(255) DEFAULT NULL,
-- ========== MODERATION DETAILS ==========
-- The last user (possibily moderator) that made changes.
-- See the future audit logs table for better info and accounting.
maybe_last_update_user_token VARCHAR(32) DEFAULT NULL,
-- ========== TIMESTAMPS ==========
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- If this is removed by a mod or the creator.
user_deleted_at TIMESTAMP NULL,
mod_deleted_at TIMESTAMP NULL,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (token),
KEY fk_model_token (model_token),
KEY fk_maybe_vocoder_token (maybe_vocoder_token),
KEY fk_media_token (media_token),
KEY fk_maybe_creator_user_token (maybe_creator_user_token),
KEY fk_maybe_creator_anonymous_visitor_token (maybe_creator_anonymous_visitor_token),
KEY index_creator_ip_address (creator_ip_address),
KEY index_user_deleted_at(user_deleted_at),
KEY index_mod_deleted_at(mod_deleted_at),
KEY index_created_at(created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
_migrations/2022-12-15-223447_voice_conversion_result_synthetic_ids/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
DROP TABLE voice_conversion_result_synthetic_ids;
_migrations/2022-12-15-223447_voice_conversion_result_synthetic_ids/up.sql
+25
View File
@@ -0,0 +1,25 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
-- It gets unwieldy to show results without a numeric sequence.
-- We create a synthetic sequence for each user to make revisiting results easier.
CREATE TABLE voice_conversion_result_synthetic_ids (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" (PUBLIC)
user_token VARCHAR(32) NOT NULL,
-- The next result id.
-- After we use an id, we increment this counter.
next_id BIGINT(20) NOT NULL DEFAULT 0,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (user_token)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
_migrations/2022-12-29-063005_media_upload_synthetic_ids/down.sql
+5
View File
@@ -0,0 +1,5 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
DROP TABLE media_upload_synthetic_ids;
_migrations/2022-12-29-063005_media_upload_synthetic_ids/up.sql
+25
View File
@@ -0,0 +1,25 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
-- It gets unwieldy to show results without a numeric sequence.
-- We create a synthetic sequence for each user to make revisiting results easier.
CREATE TABLE media_upload_synthetic_ids (
-- Not used for anything except replication.
id BIGINT(20) NOT NULL AUTO_INCREMENT,
-- Effective "primary key" (PUBLIC)
user_token VARCHAR(32) NOT NULL,
-- The next result id.
-- After we use an id, we increment this counter.
next_id BIGINT(20) NOT NULL DEFAULT 0,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-- INDICES --
PRIMARY KEY (id),
UNIQUE KEY (user_token)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
analytics/most_popular_tts_models.sql → analytics/tts_models.sql
+94 -8
View File
@@ -3,7 +3,7 @@
-- noinspection SqlResolveForFile
--
-- Top 100 models all time by use count
-- Top 500 models all time by use count
--
select
m.token,
@@ -24,7 +24,7 @@ from (
join users as u
on u.token = m.creator_user_token
order by r.use_count desc
limit 100;
limit 500;
--
-- Same, but simpler...
@@ -32,7 +32,7 @@ order by r.use_count desc
select m.token, m.title, r.use_count from (
select model_token, count(*) as use_count from tts_results
group by model_token
order by use_count desc limit 100
order by use_count desc limit 500
) as r
join tts_models as m
on m.token = r.model_token;
@@ -46,7 +46,7 @@ select m.token, m.title, m.ietf_language_tag, r.use_count from (
where created_at > ( CURDATE() - INTERVAL 30 DAY )
group by model_token
order by use_count desc
limit 100
limit 500
) as r
join tts_models as m
on m.token = r.model_token;
@@ -69,7 +69,7 @@ order by r.use_count desc
limit 500;
--
-- Top 100 models by use count, last 5 days, not spanish, etc.
-- Top 500 models by use count, last 5 days, not spanish, etc.
-- Limited due to tmux scrollback.
--
select
@@ -93,7 +93,34 @@ from (
on u.token = m.creator_user_token
where m.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX', 'pt-BR')
order by r.use_count desc
limit 100;
limit 500;
--
-- Top 500 models by use count, last 5 days, in specific language.
-- Limited due to tmux scrollback.
--
select
m.token,
m.title,
m.ietf_language_tag,
u.username,
r.use_count,
m.user_deleted_at,
m.mod_deleted_at,
m.created_at
from (
select model_token, count(*) as use_count
from tts_results
where created_at > ( CURDATE() - INTERVAL 5 DAY )
group by model_token
) as r
join tts_models as m
on m.token = r.model_token
join users as u
on u.token = m.creator_user_token
where m.ietf_language_tag IN ('en', 'en-US', 'en-AU', 'en-CA', 'en-GB')
order by r.use_count desc
limit 500;
--
-- Most popular voices by use count over 5-day window, single language, single user.
@@ -121,7 +148,7 @@ from (
m.ietf_language_tag NOT IN ('es', 'es-419', 'es-ES', 'es-MX', 'pt-BR')
AND u.username = 'vegito1089'
order by r.use_count desc
limit 100;
limit 500;
--
-- Most popular deleted models
@@ -152,4 +179,63 @@ from (
join users as u
on u.token = m.creator_user_token
order by r.use_count desc
limit 100;
limit 500;
--
-- Most popular deleted models (only by users, not mods)
-- Mod-deleted models were probably on purpose
--
select
m.token,
m.title,
m.ietf_language_tag,
u.username,
r.use_count,
m.user_deleted_at,
m.created_at
from (
select model_token, count(*) as use_count
from tts_results
where model_token IN (
select token
from tts_models
where
user_deleted_at IS NOT NULL
)
group by model_token
) as r
join tts_models as m
on m.token = r.model_token
join users as u
on u.token = m.creator_user_token
order by r.use_count desc
limit 500;
--
-- Models uploaded recently (to check if they use the right text_pipeline_type,
-- vocoder -todo-, etc.)
--
select
m.token,
m.title,
m.text_pipeline_type,
m.ietf_language_tag,
u.username,
r.use_count,
m.user_deleted_at,
m.mod_deleted_at,
m.created_at
from (
select model_token, count(*) as use_count
from tts_results
where created_at > ( CURDATE() - INTERVAL 5 DAY )
group by model_token
) as r
join tts_models as m
on m.token = r.model_token
join users as u
on u.token = m.creator_user_token
order by m.created_at desc
limit 500;
analytics/tts_results.sql
+17
View File
@@ -31,3 +31,20 @@ from (
) as sample
where sample.is_generated_on_prem IS TRUE;
-- Find TTS results for a single model
select count(*) from tts_results
where model_token = 'TM:ztt5s1be5tq6';
-- Find TTS results for a single model that are not deleted
select count(*) from tts_results
where model_token = 'TM:ztt5s1be5tq6'
and mod_deleted_at is NULL;
-- Delete TTS results for a single model
-- Do this in short batches so a lock isn't held for prohibitively long.
update tts_results
set mod_deleted_at = NOW()
where model_token = 'TM:ztt5s1be5tq6'
and mod_deleted_at IS NULL
limit 5000;
analytics/w2l_inference_jobs.sql
+7
View File
@@ -0,0 +1,7 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
-- Pending job count
select count(*) from w2l_inference_jobs where status='pending';
analytics/w2l_template_upload_jobs.sql
+28
View File
@@ -0,0 +1,28 @@
-- noinspection SqlDialectInspectionForFile
-- noinspection SqlNoDataSourceInspectionForFile
-- noinspection SqlResolveForFile
-- Get a count of pending jobs
select count(*) from w2l_template_upload_jobs where status='pending';
-- See pending download URLs:
select download_url from w2l_template_upload_jobs where status IN ('pending', 'started', 'attempt_failed');
-- See old jobs that might be stuck as "started"
select count(*)
from w2l_template_upload_jobs
where status='started'
and created_at < ( CURDATE() - INTERVAL 1 DAY );
-- Unstick old jobs that might be stuck as "started"
update w2l_template_upload_jobs
set status='dead'
where status='started'
and created_at < ( CURDATE() - INTERVAL 1 DAY );
-- Kill pending jobs with tiktok URLs:
-- These need to be blocked in the server and job directly.
update w2l_template_upload_jobs
set status='dead'
where status='pending'
and download_url LIKE '%tiktok%';
crates/_types/buckets/Cargo.toml
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "buckets"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
name = "buckets"
path = "src/lib.rs"
[dependencies]
# Internal
crockford = { path = "../../lib/crockford" }
errors = { path = "../../lib/errors" }
# External
# (none yet)
[dev-dependencies]
# (none yet)
crates/_types/buckets/src/lib.rs
+24
View File
@@ -0,0 +1,24 @@
//! buckets
//!
//! The purpose of this library is to maintain strongly typed cloud storage buckets
//! that can be shared between projects.
//!
// Never allow these
#![forbid(private_in_public)]
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
pub mod util;
pub mod private;
pub mod public;
crates/_types/buckets/src/private/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod private_path;
crates/_types/buckets/src/private/private_path.rs
+7
View File
@@ -0,0 +1,7 @@
/// A trait implemented by all private paths.
pub trait PrivatePath {
fn is_public(&self) -> bool {
false
}
}
crates/_types/buckets/src/public/media_uploads/directory.rs
+75
View File
@@ -0,0 +1,75 @@
use crate::public::public_path::PublicPath;
use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
use crockford::crockford_entropy_lower;
// TODO: Generate these from a macro.
// TODO: Use a central path registry for quick reference
const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media_upload";
/// Directory for user media uploads.
/// Each uploaded file gets its own directory so that we can store the original
/// file alongside re-processed derivatives.
#[derive(Clone)]
pub struct MediaUploadDirectory {
object_hash: String,
directory: String,
}
impl PublicPath for MediaUploadDirectory {}
impl MediaUploadDirectory {
pub fn generate_new() -> Self {
let entropy = crockford_entropy_lower(32);
Self::from_object_hash(&entropy)
}
pub fn from_object_hash(object_hash: &str) -> Self {
// TODO: Path construction could be cleaner.
let middle = hashed_directory_path_long_string(object_hash);
let directory = format!("{}/{}{}", MEDIA_UPLOAD_DIRECTORY, middle, object_hash);
Self {
object_hash: object_hash.to_string(),
directory,
}
}
pub fn get_directory_path_str(&self) -> &str {
&self.directory
}
pub fn get_object_hash(&self) -> &str {
&self.object_hash
}
}
#[cfg(test)]
mod tests {
use crate::public::media_uploads::directory::MediaUploadDirectory;
#[test]
pub fn generate_new_entropy() {
let directory = MediaUploadDirectory::generate_new();
assert_eq!(directory.get_object_hash().len(), 32);
}
#[test]
pub fn get_directory_path_str() {
let directory = MediaUploadDirectory::from_object_hash("abcdefghijk");
assert_eq!(directory.get_directory_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk");
}
#[test]
pub fn get_directory_path_str_short_name() {
let directory = MediaUploadDirectory::from_object_hash("foo");
assert_eq!(directory.get_directory_path_str(), "/media_upload/f/o/foo");
}
#[test]
pub fn get_object_hash() {
let hash = "abcdefghijk";
let directory = MediaUploadDirectory::from_object_hash(hash);
assert_eq!(directory.get_object_hash(), hash);
}
}
crates/_types/buckets/src/public/media_uploads/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod directory;
pub mod original_file;
crates/_types/buckets/src/public/media_uploads/original_file.rs
+89
View File
@@ -0,0 +1,89 @@
use crate::public::media_uploads::directory::MediaUploadDirectory;
use crate::public::public_path::PublicPath;
use crockford::crockford_entropy_lower;
const ORIGINAL_FILE_BASENAME : &'static str = "original_upload.bin";
// TODO: Generate these from a macro.
/// The original user upload file.
/// It may have derivative files (down samples, crops, etc.) that live alongside it.
#[derive(Clone)]
pub struct MediaUploadOriginalFilePath {
directory: MediaUploadDirectory,
full_object_path: String,
}
impl PublicPath for MediaUploadOriginalFilePath {}
impl MediaUploadOriginalFilePath {
pub fn generate_new() -> Self {
let entropy = crockford_entropy_lower(32);
Self::from_object_hash(&entropy)
}
pub fn from_object_hash(hash: &str) -> Self {
// TODO: Path construction could be cleaner.
let directory = MediaUploadDirectory::from_object_hash(hash);
let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME);
Self {
directory,
full_object_path,
}
}
pub fn get_full_object_path_str(&self) -> &str {
&self.full_object_path
}
pub fn get_directory(&self) -> &MediaUploadDirectory {
&self.directory
}
pub fn get_object_hash(&self) -> &str {
&self.directory.get_object_hash()
}
pub fn get_basename() -> &'static str {
ORIGINAL_FILE_BASENAME
}
}
#[cfg(test)]
mod tests {
use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath;
#[test]
pub fn generate_new_entropy() {
let file = MediaUploadOriginalFilePath::generate_new();
assert_eq!(file.get_object_hash().len(), 32);
assert_eq!(file.get_directory().get_object_hash().len(), 32);
}
#[test]
pub fn get_full_object_path_str() {
let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk");
assert_eq!(file.get_full_object_path_str(), "/media_upload/a/b/c/d/e/abcdefghijk/original_upload.bin");
}
#[test]
pub fn get_full_object_path_str_short_name() {
let file = MediaUploadOriginalFilePath::from_object_hash("foo");
assert_eq!(file.get_full_object_path_str(), "/media_upload/f/o/foo/original_upload.bin");
}
#[test]
pub fn get_full_object_path_str_starts_with_directory() {
let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk");
let full_path = file.get_full_object_path_str();
assert!(full_path.starts_with(file.get_directory().get_directory_path_str()))
}
#[test]
pub fn get_object_hash() {
let hash = "abcdefghijk";
let file = MediaUploadOriginalFilePath::from_object_hash(hash);
assert_eq!(file.get_object_hash(), hash);
}
}
crates/_types/buckets/src/public/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod media_uploads;
pub mod public_path;
crates/_types/buckets/src/public/public_path.rs
+7
View File
@@ -0,0 +1,7 @@
/// A trait implemented by all public paths.
pub trait PublicPath {
fn is_public(&self) -> bool {
true
}
}
crates/lib/storage_buckets_common/src/bucket_paths.rs → crates/_types/buckets/src/util/hash_to_bucket_path.rs
+10 -14
View File
@@ -1,11 +1,11 @@
use anyhow::anyhow;
use anyhow::bail;
use container_common::anyhow_result::AnyhowResult;
use errors::AnyhowResult;
use errors::bail;
use std::path::PathBuf;
pub fn hash_to_bucket_path(file_hash: &str,
#[deprecated(note = "this is a messy function and should be avoided")]
pub (crate) fn hash_to_bucket_path(file_hash: &str,
optional_root_directory: Option<&str>)
-> AnyhowResult<String>
-> AnyhowResult<PathBuf>
{
if file_hash.len() < 4 {
bail!("File length is too short");
@@ -23,29 +23,25 @@ pub fn hash_to_bucket_path(file_hash: &str,
&file_hash,
);
let maybe_path = root_directory
.join(file_path)
.to_str()
.map(|s| s.to_string());
maybe_path.ok_or(anyhow!("Path could not be converted"))
Ok(root_directory.join(file_path))
}
#[cfg(test)]
mod tests {
use crate::bucket_paths::hash_to_bucket_path;
use crate::util::hash_to_bucket_path::hash_to_bucket_path;
use std::path::PathBuf;
#[test]
fn test_bucket_without_root() {
assert_eq!(
hash_to_bucket_path("abcdefg", None).unwrap(),
"/a/b/c/abcdefg".to_string());
PathBuf::from("/a/b/c/abcdefg"));
}
#[test]
fn test_bucket_with_root() {
assert_eq!(
hash_to_bucket_path("abcdefg", Some("/root")).unwrap(),
"/root/a/b/c/abcdefg".to_string());
PathBuf::from("/root/a/b/c/abcdefg"));
}
}
crates/_types/buckets/src/util/hash_to_bucket_path_string.rs
+34
View File
@@ -0,0 +1,34 @@
use crate::util::hash_to_bucket_path::hash_to_bucket_path;
use errors::AnyhowResult;
use errors::anyhow;
#[deprecated(note = "this is a messy function and should be avoided")]
pub fn hash_to_bucket_path_string(file_hash: &str,
optional_root_directory: Option<&str>)
-> AnyhowResult<String>
{
let path = hash_to_bucket_path(file_hash, optional_root_directory)?;
let maybe_path = path.to_str().map(|s| s.to_string());
maybe_path.ok_or(anyhow!("Path could not be converted"))
}
#[cfg(test)]
mod tests {
use crate::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
#[test]
fn test_bucket_without_root() {
assert_eq!(
hash_to_bucket_path_string("abcdefg", None).unwrap(),
"/a/b/c/abcdefg".to_string());
}
#[test]
fn test_bucket_with_root() {
assert_eq!(
hash_to_bucket_path_string("abcdefg", Some("/root")).unwrap(),
"/root/a/b/c/abcdefg".to_string());
}
}
crates/_types/buckets/src/util/hashed_directory_path_long_string.rs
+67
View File
@@ -0,0 +1,67 @@
/// Create a cloud object store directory structure that can be easily traversed without
/// too many items living in a single directory.
///
/// With a million items, each directory will have ~1 item.
/// With a billion items, each directory will have ~953 items (totally tractable).
///
pub fn hashed_directory_path_long_string(file_hash: &str) -> String {
match file_hash.len() {
0 | 1 => "".to_string(),
2 => format!("{}/", &file_hash[0..1]),
3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
4 => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
5 => format!("{}/{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3], &file_hash[3..4]),
_ => format!("{}/{}/{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3], &file_hash[3..4], &file_hash[4..5]),
}
}
#[cfg(test)]
mod tests {
use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
#[test]
fn test_length_zero() {
assert_eq!(hashed_directory_path_long_string(""), "".to_string());
}
#[test]
fn test_length_one() {
assert_eq!(hashed_directory_path_long_string("a"), "".to_string());
}
#[test]
fn test_length_two() {
assert_eq!(hashed_directory_path_long_string("ab"), "a/".to_string());
}
#[test]
fn test_length_three() {
assert_eq!(hashed_directory_path_long_string("abc"), "a/b/".to_string());
}
#[test]
fn test_length_four() {
assert_eq!(hashed_directory_path_long_string("abcd"), "a/b/c/".to_string());
}
#[test]
fn test_length_five() {
assert_eq!(hashed_directory_path_long_string("abcde"), "a/b/c/d/".to_string());
}
#[test]
fn test_length_six() {
assert_eq!(hashed_directory_path_long_string("abcdef"), "a/b/c/d/e/".to_string());
}
#[test]
fn test_length_seven() {
assert_eq!(hashed_directory_path_long_string("abcdefg"), "a/b/c/d/e/".to_string());
}
#[test]
fn test_length_ten() {
assert_eq!(hashed_directory_path_long_string("abcdefghij"), "a/b/c/d/e/".to_string());
}
}
crates/_types/buckets/src/util/hashed_directory_path_short_string.rs
+55
View File
@@ -0,0 +1,55 @@
/// Create a cloud object store directory structure that can be easily traversed without
/// too many items living in a single directory.
///
/// With a million items, each directory will have ~244 items.
/// With a billion items, each directory will have ~244,140 items (impractical at this scale).
///
pub fn hashed_directory_path_short_string(file_hash: &str) -> String {
match file_hash.len() {
0 | 1 => "".to_string(),
2 => format!("{}/", &file_hash[0..1]),
3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
_ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
}
}
#[cfg(test)]
mod tests {
use crate::util::hashed_directory_path_short_string::hashed_directory_path_short_string;
#[test]
fn test_length_zero() {
assert_eq!(hashed_directory_path_short_string(""), "".to_string());
}
#[test]
fn test_length_one() {
assert_eq!(hashed_directory_path_short_string("a"), "".to_string());
}
#[test]
fn test_length_two() {
assert_eq!(hashed_directory_path_short_string("ab"), "a/".to_string());
}
#[test]
fn test_length_three() {
assert_eq!(hashed_directory_path_short_string("abc"), "a/b/".to_string());
}
#[test]
fn test_length_four() {
assert_eq!(hashed_directory_path_short_string("abcd"), "a/b/c/".to_string());
}
#[test]
fn test_length_five() {
assert_eq!(hashed_directory_path_short_string("abcde"), "a/b/c/".to_string());
}
#[test]
fn test_length_ten() {
assert_eq!(hashed_directory_path_short_string("abcdefghij"), "a/b/c/".to_string());
}
}
crates/_types/buckets/src/util/mod.rs
+4
View File
@@ -0,0 +1,4 @@
pub (crate) mod hash_to_bucket_path;
pub mod hash_to_bucket_path_string;
pub mod hashed_directory_path_long_string;
pub mod hashed_directory_path_short_string;
crates/_types/enums/Cargo.toml
+33
View File
@@ -0,0 +1,33 @@
[package]
name = "enums"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
name = "enums"
path = "src/lib.rs"
[dependencies]
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.64"
toml = "0.5.8"
# MySql Database
# "offline" supports typesafe builds without a live database
# "rustls" doesn't appear to work with IP addresses, per:
# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
#sqlx = { version = "0.4", features = [ "mysql", "runtime-tokio-native-tls", "offline", "chrono" ] }
sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
sqlx-core = { version = "0.5.7" }
[dev-dependencies]
# Enum utilities
strum = { version = "0.24", features = ["derive"] }
crates/_types/enums/src/core/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod visibility;
crates/_types/enums/src/core/visibility.rs
+189
View File
@@ -0,0 +1,189 @@
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
/// Visibility
///
/// Used in various database tables (as enums! careful!) and the HTTP API to convey
/// how the associated entity should be made visible to the public.
///
/// To use this in a query, the query must have type annotations.
/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375
/// eg. preferred_tts_result_visibility as `preferred_tts_result_visibility: enums::core::visibility::Visibility`
///
/// See also: https://docs.rs/sqlx/0.4.0-beta.1/sqlx/trait.Type.html
///
/// *DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY!*
///
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize, sqlx::Type)]
#[serde(rename_all = "lowercase")]
#[sqlx(rename_all = "lowercase")]
pub enum Visibility {
/// Public entities are able to be listed in public lists.
/// It does not mean that they necessarily will be (eg. they could be "mod unapproved" or deleted).
Public,
/// Hidden entities are not shown in public lists, but the URL to them may be given out freely.
/// They are available to non-logged-in users as long as they have the URL.
Hidden,
/// Private entities should only be available to the creator, a list of approved users, and
/// website moderation staff.
Private,
// TODO(bt, 2022-12-20): We need a "Shared" option where users can share it with a specified group.
// This should perhaps be its own type, eg. VisibilityV2., so that we don't use it in tables that
// have not yet been migrated to this scheme.
}
impl_enum_display_and_debug_using_to_str!(Visibility);
// For reference, here's what the serde implementation might be if manually written.
// This may be useful for designing composite types in the future:
//
// use serde::{Deserializer, Serializer};
//
// impl serde::Serialize for UserToken {
// fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
// serializer.serialize_str(&self.0)
// }
// }
//
// impl<'de> serde::Deserialize<'de> for UserToken {
// fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
// let s = String::deserialize(d)?;
// Ok(UserToken(s))
// }
// }
//impl sqlx::Type<MySql> for Visibility {
// fn type_info() -> sqlx_core::database::TypeInfo<MySql> {
// todo!()
// }
//}
impl Default for Visibility {
fn default() -> Self { Self::Public }
}
/// NB: Legacy API for older code.
impl Visibility {
pub fn to_str(&self) -> &'static str {
match self {
Self::Public => "public",
Self::Hidden => "hidden",
Self::Private => "private",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"public" => Ok(Self::Public),
"hidden" => Ok(Self::Hidden),
"private" => Ok(Self::Private),
_ => Err(format!("invalid value: {:?}", value)),
}
}
}
#[cfg(test)]
mod tests {
use crate::core::visibility::Visibility;
use crate::test_helpers::assert_serialization;
#[test]
fn test_default() {
assert_eq!(Visibility::default(), Visibility::Public);
}
#[test]
fn test_serialization() {
assert_serialization(Visibility::Public, "public");
assert_serialization(Visibility::Hidden, "hidden");
assert_serialization(Visibility::Private, "private");
}
#[test]
fn test_to_str() {
assert_eq!(Visibility::Public.to_str(), "public");
assert_eq!(Visibility::Hidden.to_str(), "hidden");
assert_eq!(Visibility::Private.to_str(), "private");
}
#[test]
fn test_from_str() {
assert_eq!(Visibility::from_str("public").unwrap(), Visibility::Public);
assert_eq!(Visibility::from_str("hidden").unwrap(), Visibility::Hidden);
assert_eq!(Visibility::from_str("private").unwrap(), Visibility::Private);
assert!(Visibility::from_str("foo").is_err());
}
mod traits {
use crate::core::visibility::Visibility;
#[test]
fn display() {
let visibility = Visibility::Hidden;
assert_eq!(format!("{}", visibility), "hidden".to_string());
}
#[test]
fn debug() {
let visibility = Visibility::Private;
assert_eq!(format!("{:?}", visibility), "private".to_string());
}
}
#[derive(Serialize, Deserialize, PartialEq, Debug)]
struct CompositeType {
visibility: Visibility,
string: String,
}
mod serde_serialization {
use crate::core::visibility::Visibility;
use super::CompositeType;
#[test]
fn serialize() {
let expected = "\"public\"".to_string(); // NB: Quoted
assert_eq!(expected, toml::to_string(&Visibility::Public).unwrap());
// Just to show this serializes the same as a string
assert_eq!(expected, toml::to_string("public").unwrap());
}
#[test]
fn nested_serialize() {
let value = CompositeType { visibility: Visibility::Hidden, string: "bar".to_string() };
let expected = r#"{"visibility":"hidden","string":"bar"}"#.to_string();
assert_eq!(expected, serde_json::to_string(&value).unwrap());
}
}
mod serde_deserialization {
use crate::core::visibility::Visibility;
use super::CompositeType;
#[test]
fn deserialize() {
let payload = "\"private\""; // NB: Quoted
let value: Visibility = serde_json::from_str(payload).unwrap();
assert_eq!(value, Visibility::Private);
}
#[test]
fn nested_deserialize() {
let payload = r#"{"visibility":"hidden","string":"bar"}"#.to_string();
let expected = CompositeType {
visibility: Visibility::Hidden,
string: "bar".to_string(),
};
assert_eq!(expected, serde_json::from_str::<CompositeType>(&payload).unwrap());
}
}
}
crates/_types/enums/src/files/media_upload_type.rs
+67
View File
@@ -0,0 +1,67 @@
use serde::Deserialize;
use serde::Serialize;
#[cfg(test)]
use strum::EnumCount;
#[cfg(test)]
use strum::EnumIter;
/// Used in the `media_uploads` table in a `VARCHAR` field.
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum MediaUploadType {
/// Audio files: wav, mp3, etc.
Audio,
/// Video files: mp4, etc.
Video,
}
// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests!
impl_enum_display_and_debug_using_to_str!(MediaUploadType);
impl_enum_sqlx_coders!(MediaUploadType);
/// NB: Legacy API for older code.
impl MediaUploadType {
pub fn to_str(&self) -> &'static str {
match self {
Self::Audio => "audio",
Self::Video => "video",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"audio" => Ok(Self::Audio),
"video" => Ok(Self::Video),
_ => Err(format!("invalid value: {:?}", value)),
}
}
}
#[cfg(test)]
mod tests {
use crate::files::media_upload_type::MediaUploadType;
use crate::test_helpers::assert_serialization;
#[test]
fn test_serialization() {
assert_serialization(MediaUploadType::Audio, "audio");
assert_serialization(MediaUploadType::Video, "video");
}
#[test]
fn test_to_str() {
assert_eq!(MediaUploadType::Audio.to_str(), "audio");
assert_eq!(MediaUploadType::Video.to_str(), "video");
}
#[test]
fn test_from_str() {
assert_eq!(MediaUploadType::from_str("audio").unwrap(), MediaUploadType::Audio);
assert_eq!(MediaUploadType::from_str("video").unwrap(), MediaUploadType::Video);
assert!(MediaUploadType::from_str("foo").is_err());
}
}
crates/_types/enums/src/files/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod media_upload_type;
crates/_types/enums/src/lib.rs
+32
View File
@@ -0,0 +1,32 @@
//! enums
//!
//! The purpose of this library is to have a strongly-typed MySQL enum-type wrapper.
//! This should also work for CHAR/VARCHAR fields that work similarly to enums (typically
//! as part of a composite key)
//!
// Never allow these
#![forbid(private_in_public)]
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
#[macro_use] extern crate serde_derive;
#[macro_use]
mod macros;
#[cfg(test)] pub mod test_helpers;
pub mod core;
pub mod files;
pub mod ml;
pub mod workers;
crates/_types/enums/src/macros.rs
+86
View File
@@ -0,0 +1,86 @@
/// Implement `Display` and `Debug` for enums that have a `.to_str()` method.
/// This ensures that the casing follows whatever `.to_str()` specifies.
macro_rules! impl_enum_display_and_debug_using_to_str {
($t:ident) => {
// Debug trait.
// Requires that the type has `.to_str()`.
impl std::fmt::Debug for $t {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.to_str())
}
}
// Display trait.
// Requires that the type has `.to_str()`.
impl std::fmt::Display for $t {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.to_str())
}
}
// NB: This test requires `strum::{EnumIter}` on the enum
#[cfg(test)]
#[test]
fn test_display_trait_matches_to_str() {
use strum::IntoEnumIterator;
for variant in $t::iter() {
assert_eq!(format!("{}", variant), variant.to_str());
}
}
// NB: This test requires `strum::{EnumIter}` on the enum
#[cfg(test)]
#[test]
fn test_debug_trait_matches_to_str() {
use strum::IntoEnumIterator;
for variant in $t::iter() {
assert_eq!(format!("{:?}", variant), variant.to_str());
}
}
}
}
/// This overt approach is being taken because of the following error:
///
/// `MySqlDatabaseError { code: Some("HY000"), number: 1210, message: "Incorrect arguments to mysqld_stmt_execute" }`
///
/// Basically, sqlx can't turn our enum into a VARCHAR when using #[derive(sqlx::Type)].
/// Unfortunately, by not using this, we also lose the ability to `#[sqlx(rename_all="lowercase")]`, etc.,
/// so our encoder/decoder need to set the rules.
///
/// Solution adapted from https://github.com/launchbadge/sqlx/discussions/1502
macro_rules! impl_enum_sqlx_coders {
($t:ident) => {
impl sqlx::Type<sqlx_core::mysql::MySql> for $t {
fn type_info() -> sqlx_core::mysql::MySqlTypeInfo {
String::type_info()
}
}
impl<'q> sqlx::Encode<'q, sqlx_core::mysql::MySql> for $t {
fn encode_by_ref(
&self,
buf: &mut <sqlx_core::mysql::MySql as sqlx_core::database::HasArguments<'q>>::ArgumentBuffer
) -> sqlx_core::encode::IsNull {
// NB: In the absence of `#[derive(sqlx::Type)]` and `#sqlx(rename_all="lowercase")]`,
// this controls the casing of the variants when sent to MySQL.
self.to_str().encode_by_ref(buf)
}
}
impl<'r> sqlx::Decode<'r, sqlx_core::mysql::MySql> for $t {
fn decode(
value: sqlx_core::mysql::MySqlValueRef<'r>,
) -> Result<Self, sqlx_core::error::BoxDynError> {
let string = String::decode(value)?;
let value = $t::from_str(&string)?;
Ok(value)
}
}
}
}
crates/_types/enums/src/ml/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod vocoder_type;
crates/_types/enums/src/ml/vocoder_type.rs
+56
View File
@@ -0,0 +1,56 @@
#[derive(Clone, Copy, Eq, PartialEq, Debug, Deserialize, Serialize, sqlx::Type)]
pub enum VocoderType {
/// NB: Note - this is hifigan for Tacotron2.
/// Some work will be needed to unify this with other hifigan types.
#[serde(rename = "hifigan")]
#[sqlx(rename = "hifigan")]
HifiGan,
/// NB: Note - this is hifigan for SoftVC (our internal codename is "rocketvc").
/// Some work will need to be done to unify this with other hifigan types.
#[serde(rename = "hifigan_rocket_vc")]
#[sqlx(rename = "hifigan_rocket_vc")]
HifiGanRocketVc,
}
/// NB: Legacy API for older code.
impl VocoderType {
pub fn to_str(&self) -> &'static str {
match self {
Self::HifiGan=> "hifigan",
Self::HifiGanRocketVc => "hifigan_rocket_vc",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"hifigan" => Ok(Self::HifiGan),
"hifigan_rocket_vc" => Ok(Self::HifiGanRocketVc),
_ => Err(format!("invalid value: {:?}", value)),
}
}
}
#[cfg(test)]
mod tests {
use crate::ml::vocoder_type::VocoderType;
use crate::test_helpers::assert_serialization;
#[test]
fn test_serialization() {
assert_serialization(VocoderType::HifiGan, "hifigan");
assert_serialization(VocoderType::HifiGanRocketVc, "hifigan_rocket_vc");
}
#[test]
fn to_str() {
assert_eq!(VocoderType::HifiGan.to_str(), "hifigan");
assert_eq!(VocoderType::HifiGanRocketVc.to_str(), "hifigan_rocket_vc");
}
#[test]
fn from_str() {
assert_eq!(VocoderType::from_str("hifigan").unwrap(), VocoderType::HifiGan);
assert_eq!(VocoderType::from_str("hifigan_rocket_vc").unwrap(), VocoderType::HifiGanRocketVc);
}
}
crates/lib/reusable_types/src/test_helpers.rs → crates/_types/enums/src/test_helpers.rs
View File
File renamed without changes.
crates/lib/reusable_types/src/generic_download_type.rs → crates/_types/enums/src/workers/generic_download_type.rs
+34 -9
View File
@@ -1,18 +1,38 @@
// TODO: Use macro-derived impls
/// Our "generic downloads" pipeline supports a wide variety of ML models and other media.
/// They are serialized in the database table `generic_download_jobs` as a VARCHAR(32).
///
/// Each type of download is identified by the following enum variants.
/// These types are present in the HTTP API and database columns as serialized here.
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[derive(Clone, Copy, Eq, PartialEq, Debug, Deserialize, Serialize, sqlx::Type)]
pub enum GenericDownloadType {
/// NB: Note - this is hifigan for Tacotron2.
/// Some work will be needed to unify this with other hifigan types.
#[serde(rename = "hifigan")]
#[sqlx(rename = "hifigan")]
HifiGan,
#[serde(rename = "melgan_vocodes")]
#[sqlx(rename = "melgan_vocodes")]
MelGanVocodes,
/// NB: Note - this is hifigan for SoftVC (our internal codename is "rocketvc").
/// Some work will need to be done to unify this with other hifigan types.
#[serde(rename = "hifigan_rocket_vc")]
#[sqlx(rename = "hifigan_rocket_vc")]
HifiGanRocketVc,
//#[serde(rename = "melgan_vocodes")]
//#[sqlx(rename = "melgan_vocodes")]
//MelGanVocodes,
/// NB: Our external-facing name for "softvc" is rocketvc.
/// I wish we could stop being stupid about this.
#[serde(rename = "rocket_vc")]
#[sqlx(rename = "rocket_vc")]
RocketVc,
/// Tacotron TTS models.
#[serde(rename = "tacotron2")]
#[sqlx(rename = "tacotron2")]
Tacotron2,
@@ -23,7 +43,8 @@ impl GenericDownloadType {
pub fn to_str(&self) -> &'static str {
match self {
Self::HifiGan => "hifigan",
Self::MelGanVocodes => "melgan_vocodes",
Self::HifiGanRocketVc => "hifigan_rocket_vc",
Self::RocketVc => "rocket_vc",
Self::Tacotron2 => "tacotron2",
}
}
@@ -31,7 +52,8 @@ impl GenericDownloadType {
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"hifigan" => Ok(Self::HifiGan),
"melgan_vocodes" => Ok(Self::MelGanVocodes),
"hifigan_rocket_vc" => Ok(Self::HifiGanRocketVc),
"rocket_vc" => Ok(Self::RocketVc),
"tacotron2" => Ok(Self::Tacotron2),
_ => Err(format!("invalid value: {:?}", value)),
}
@@ -40,27 +62,30 @@ impl GenericDownloadType {
#[cfg(test)]
mod tests {
use crate::generic_download_type::GenericDownloadType;
use crate::workers::generic_download_type::GenericDownloadType;
use crate::test_helpers::assert_serialization;
#[test]
fn test_serialization() {
assert_serialization(GenericDownloadType::HifiGan, "hifigan");
assert_serialization(GenericDownloadType::MelGanVocodes, "melgan_vocodes");
assert_serialization(GenericDownloadType::HifiGanRocketVc, "hifigan_rocket_vc");
assert_serialization(GenericDownloadType::RocketVc, "rocket_vc");
assert_serialization(GenericDownloadType::Tacotron2, "tacotron2");
}
#[test]
fn to_str() {
assert_eq!(GenericDownloadType::HifiGan.to_str(), "hifigan");
assert_eq!(GenericDownloadType::MelGanVocodes.to_str(), "melgan_vocodes");
assert_eq!(GenericDownloadType::HifiGanRocketVc.to_str(), "hifigan_rocket_vc");
assert_eq!(GenericDownloadType::RocketVc.to_str(), "rocket_vc");
assert_eq!(GenericDownloadType::Tacotron2.to_str(), "tacotron2");
}
#[test]
fn from_str() {
assert_eq!(GenericDownloadType::from_str("hifigan").unwrap(), GenericDownloadType::HifiGan);
assert_eq!(GenericDownloadType::from_str("melgan_vocodes").unwrap(), GenericDownloadType::MelGanVocodes);
assert_eq!(GenericDownloadType::from_str("hifigan_rocket_vc").unwrap(), GenericDownloadType::HifiGanRocketVc);
assert_eq!(GenericDownloadType::from_str("rocket_vc").unwrap(), GenericDownloadType::RocketVc);
assert_eq!(GenericDownloadType::from_str("tacotron2").unwrap(), GenericDownloadType::Tacotron2);
}
}
crates/_types/enums/src/workers/generic_inference_type.rs
+80
View File
@@ -0,0 +1,80 @@
// TODO: Use macro-derived impls
use std::collections::BTreeSet;
/// Our "generic inference" pipeline supports a wide variety of ML models and other media.
/// Each type of inference is identified by the following enum variants.
/// These types are present in the HTTP API and database columns as serialized here.
///
/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY.
#[derive(Clone, Copy, Eq, PartialEq, Debug, Hash, Ord, PartialOrd, Deserialize, Serialize, sqlx::Type)]
pub enum GenericInferenceType {
#[serde(rename = "text_to_speech")]
#[sqlx(rename = "text_to_speech")]
TextToSpeech,
#[serde(rename = "voice_conversion")]
#[sqlx(rename = "voice_conversion")]
VoiceConversion,
}
/// NB: Legacy API for older code.
impl GenericInferenceType {
pub fn to_str(&self) -> &'static str {
match self {
Self::TextToSpeech => "text_to_speech",
Self::VoiceConversion => "voice_conversion",
}
}
pub fn from_str(value: &str) -> Result<Self, String> {
match value {
"text_to_speech" => Ok(Self::TextToSpeech),
"voice_conversion" => Ok(Self::VoiceConversion),
_ => Err(format!("invalid value: {:?}", value)),
}
}
pub fn all_variants() -> BTreeSet<GenericInferenceType> {
// NB: BTreeSet is sorted
// NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc.
BTreeSet::from([
GenericInferenceType::TextToSpeech,
GenericInferenceType::VoiceConversion,
])
}
}
#[cfg(test)]
mod tests {
use crate::workers::generic_inference_type::GenericInferenceType;
use crate::test_helpers::assert_serialization;
#[test]
fn test_serialization() {
assert_serialization(GenericInferenceType::TextToSpeech, "text_to_speech");
assert_serialization(GenericInferenceType::VoiceConversion, "voice_conversion");
}
#[test]
fn to_str() {
assert_eq!(GenericInferenceType::TextToSpeech.to_str(), "text_to_speech");
assert_eq!(GenericInferenceType::VoiceConversion.to_str(), "voice_conversion");
}
#[test]
fn from_str() {
assert_eq!(GenericInferenceType::from_str("text_to_speech").unwrap(), GenericInferenceType::TextToSpeech);
assert_eq!(GenericInferenceType::from_str("voice_conversion").unwrap(), GenericInferenceType::VoiceConversion);
}
#[test]
fn all_variants() {
let mut variants = GenericInferenceType::all_variants();
assert_eq!(variants.len(), 2);
assert_eq!(variants.pop_first(), Some(GenericInferenceType::TextToSpeech));
assert_eq!(variants.pop_first(), Some(GenericInferenceType::VoiceConversion));
assert_eq!(variants.pop_first(), None);
}
}
crates/_types/enums/src/workers/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod generic_inference_type;
pub mod generic_download_type;
crates/lib/reusable_types/Cargo.toml → crates/_types/reusable_types/Cargo.toml
View File
File renamed without changes.
crates/lib/reusable_types/src/achievements/mod.rs → crates/_types/reusable_types/src/achievements/mod.rs
View File
File renamed without changes.
crates/lib/reusable_types/src/achievements/user_badge.rs → crates/_types/reusable_types/src/achievements/user_badge.rs
+58 -2
View File
@@ -17,6 +17,8 @@ pub enum UserBadge {
TtsModelUploader150,
TtsModelUploader200,
TtsModelUploader250,
TtsModelUploader500,
TtsModelUploader1000,
// Granted for vocoder models
VocoderModelUploader1,
@@ -28,6 +30,21 @@ pub enum UserBadge {
VocoderModelUploader150,
VocoderModelUploader200,
VocoderModelUploader250,
VocoderModelUploader500,
VocoderModelUploader1000,
// Granted for vocoder (softvc aka "rocket_vc") models
VocoderRocketVcModelUploader1,
VocoderRocketVcModelUploader5,
VocoderRocketVcModelUploader10,
VocoderRocketVcModelUploader20,
VocoderRocketVcModelUploader50,
VocoderRocketVcModelUploader100,
VocoderRocketVcModelUploader150,
VocoderRocketVcModelUploader200,
VocoderRocketVcModelUploader250,
VocoderRocketVcModelUploader500,
VocoderRocketVcModelUploader1000,
// Granted for uploading templates
W2lTemplateUploader1,
@@ -57,6 +74,7 @@ pub enum UserBadge {
}
impl UserBadge {
/// This is the value that lives in the database as a VARCHAR(32)
pub fn to_db_value(&self) -> &'static str {
match self {
UserBadge::EarlyUser => "early_user",
@@ -69,6 +87,9 @@ impl UserBadge {
UserBadge::TtsModelUploader150 => "tts_model_uploader_150",
UserBadge::TtsModelUploader200 => "tts_model_uploader_200",
UserBadge::TtsModelUploader250 => "tts_model_uploader_250",
UserBadge::TtsModelUploader500 => "tts_model_uploader_500",
UserBadge::TtsModelUploader1000 => "tts_model_uploader_1000",
UserBadge::VocoderModelUploader1 => "vocoder_model_uploader_1",
UserBadge::VocoderModelUploader5 => "vocoder_model_uploader_5",
UserBadge::VocoderModelUploader10 => "vocoder_model_uploader_10",
@@ -78,6 +99,21 @@ impl UserBadge {
UserBadge::VocoderModelUploader150 => "vocoder_model_uploader_150",
UserBadge::VocoderModelUploader200 => "vocoder_model_uploader_200",
UserBadge::VocoderModelUploader250 => "vocoder_model_uploader_250",
UserBadge::VocoderModelUploader500 => "vocoder_model_uploader_500",
UserBadge::VocoderModelUploader1000 => "vocoder_model_uploader_1000",
UserBadge::VocoderRocketVcModelUploader1 => "vocoder_rocket_vc_uploader_1",
UserBadge::VocoderRocketVcModelUploader5 => "vocoder_rocket_vc_uploader_5",
UserBadge::VocoderRocketVcModelUploader10 => "vocoder_rocket_vc_uploader_10",
UserBadge::VocoderRocketVcModelUploader20 => "vocoder_rocket_vc_uploader_20",
UserBadge::VocoderRocketVcModelUploader50 => "vocoder_rocket_vc_uploader_50",
UserBadge::VocoderRocketVcModelUploader100 => "vocoder_rocket_vc_uploader_100",
UserBadge::VocoderRocketVcModelUploader150 => "vocoder_rocket_vc_uploader_150",
UserBadge::VocoderRocketVcModelUploader200 => "vocoder_rocket_vc_uploader_200",
UserBadge::VocoderRocketVcModelUploader250 => "vocoder_rocket_vc_uploader_250",
UserBadge::VocoderRocketVcModelUploader500 => "vocoder_rocket_vc_uploader_500",
UserBadge::VocoderRocketVcModelUploader1000 => "vocoder_rocket_vc_uploader_1000", // NB: 31 characters, table max is 32
UserBadge::W2lTemplateUploader1 => "w2l_template_uploader_1",
UserBadge::W2lTemplateUploader10 => "w2l_template_uploader_10",
UserBadge::W2lTemplateUploader50 => "w2l_template_uploader_50",
@@ -88,6 +124,7 @@ impl UserBadge {
UserBadge::W2lTemplateUploader2000 => "w2l_template_uploader_2000",
UserBadge::W2lTemplateUploader5000 => "w2l_template_uploader_5000",
UserBadge::W2lTemplateUploader10000 => "w2l_template_uploader_10000",
//UserBadge::TtsInferenceUser100 => "tts_inference_100",
//UserBadge::TtsInferenceUser500 => "tts_inference_500",
//UserBadge::TtsInferenceUser1000 => "tts_inference_1000",
@@ -113,7 +150,10 @@ impl UserBadge {
| UserBadge::TtsModelUploader100
| UserBadge::TtsModelUploader150
| UserBadge::TtsModelUploader200
| UserBadge::TtsModelUploader250 => UserBadgeType::TtsModelUploader,
| UserBadge::TtsModelUploader250
| UserBadge::TtsModelUploader500
| UserBadge::TtsModelUploader1000
=> UserBadgeType::TtsModelUploader,
UserBadge::VocoderModelUploader1
| UserBadge::VocoderModelUploader5
@@ -123,7 +163,23 @@ impl UserBadge {
| UserBadge::VocoderModelUploader100
| UserBadge::VocoderModelUploader150
| UserBadge::VocoderModelUploader200
| UserBadge::VocoderModelUploader250 => UserBadgeType::VocoderModelUploader,
| UserBadge::VocoderModelUploader250
| UserBadge::VocoderModelUploader500
| UserBadge::VocoderModelUploader1000
=> UserBadgeType::VocoderModelUploader,
UserBadge::VocoderRocketVcModelUploader1
| UserBadge::VocoderRocketVcModelUploader5
| UserBadge::VocoderRocketVcModelUploader10
| UserBadge::VocoderRocketVcModelUploader20
| UserBadge::VocoderRocketVcModelUploader50
| UserBadge::VocoderRocketVcModelUploader100
| UserBadge::VocoderRocketVcModelUploader150
| UserBadge::VocoderRocketVcModelUploader200
| UserBadge::VocoderRocketVcModelUploader250
| UserBadge::VocoderRocketVcModelUploader500
| UserBadge::VocoderRocketVcModelUploader1000
=> UserBadgeType::VocoderRocketVcModelUploader,
UserBadge::W2lTemplateUploader1
| UserBadge::W2lTemplateUploader10
crates/lib/reusable_types/src/achievements/user_badge_type.rs → crates/_types/reusable_types/src/achievements/user_badge_type.rs
+3
View File
@@ -11,6 +11,9 @@ pub enum UserBadgeType {
/// Granted for uploading vocoder models
VocoderModelUploader,
/// Granted for uploading vocoder (softvc aka "rocket_vc") models
VocoderRocketVcModelUploader,
/// Granted for uploading W2L templates
W2lTemplateUploader,
crates/lib/reusable_types/src/lib.rs → crates/_types/reusable_types/src/lib.rs
-3
View File
@@ -17,8 +17,5 @@
#[cfg(test)] pub mod test_helpers;
pub mod achievements;
pub mod entity_visibility;
pub mod generic_download_type;
pub mod server_environment;
pub mod stripe;
pub mod vocoder_type;
crates/lib/reusable_types/src/server_environment.rs → crates/_types/reusable_types/src/server_environment.rs
View File
File renamed without changes.
crates/lib/reusable_types/src/stripe/mod.rs → crates/_types/reusable_types/src/stripe/mod.rs
View File
File renamed without changes.
crates/lib/reusable_types/src/stripe/stripe_recurring_interval.rs → crates/_types/reusable_types/src/stripe/stripe_recurring_interval.rs
View File
File renamed without changes.
crates/lib/reusable_types/src/stripe/stripe_subscription_status.rs → crates/_types/reusable_types/src/stripe/stripe_subscription_status.rs
View File
File renamed without changes.
crates/_types/reusable_types/src/test_helpers.rs
+16
View File
@@ -0,0 +1,16 @@
//! Only imported in tests
use serde::Serialize;
pub fn to_json<T: Serialize>(t: T) -> String {
toml::to_string(&t)
.expect("serialization error")
.replace("\"", "") // JSON values are quoted, so we remove quotes
}
/// Assert that the Serialize is represented by the expected string value.
/// This is useful for ensuring stability of serialization.
pub fn assert_serialization<T: Serialize>(t: T, expected: &str) {
// TODO: See if there's a way to check the sqlx serialization
assert_eq!(&to_json(t), expected);
}
crates/lib/tokens/Cargo.toml → crates/_types/tokens/Cargo.toml
-3
View File
@@ -13,9 +13,6 @@ name = "tokens"
path = "src/lib.rs"
[dependencies]
# Errors
anyhow = "1.0.40"
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
crates/_types/tokens/src/files/media_upload.rs
+12
View File
@@ -0,0 +1,12 @@
use crate::prefixes::EntityType;
use serde::Deserialize;
use serde::Serialize;
use std::fmt::Debug;
/// The primary key for user media uploads (images, video, etc.)
#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
#[sqlx(transparent)]
pub struct MediaUploadToken(pub String);
impl_string_token!(MediaUploadToken);
impl_crockford_generator!(MediaUploadToken, 32usize, EntityType::MediaUpload, CrockfordLower);
crates/_types/tokens/src/files/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod media_upload;
crates/lib/tokens/src/jobs/download.rs → crates/_types/tokens/src/jobs/download.rs
View File
File renamed without changes.
crates/lib/tokens/src/jobs/inference.rs → crates/_types/tokens/src/jobs/inference.rs
View File
File renamed without changes.
crates/lib/tokens/src/jobs/mod.rs → crates/_types/tokens/src/jobs/mod.rs
View File
File renamed without changes.
crates/lib/tokens/src/lib.rs → crates/_types/tokens/src/lib.rs
+4
View File
@@ -1,7 +1,9 @@
//! tokens
//!
//! The purpose of this library is to have a strongly-typed primary/foreign key system.
//! Every Database or Redis key will have a type here.
//! Well known keys will have short identifiers (eg. "user" is prefixed with "U:")
//!
// Never allow these
#![forbid(private_in_public)]
@@ -37,5 +39,7 @@ mod macros;
pub (crate) mod prefixes;
pub mod files;
pub mod jobs;
pub mod users;
pub mod voice_conversion;
crates/lib/tokens/src/macros.rs → crates/_types/tokens/src/macros.rs
View File
File renamed without changes.
crates/lib/tokens/src/prefixes.rs → crates/_types/tokens/src/prefixes.rs
+19 -1
View File
@@ -15,16 +15,20 @@ use strum::EnumIter;
pub(crate) enum EntityType {
DownloadJob,
InferenceJob,
MediaUpload,
User,
VoiceConversionModel,
}
impl EntityType {
pub fn prefix(self) -> &'static str {
match self {
Self::DownloadJob => "jdown_:", // NB: Was "JGUP:"
Self::DownloadJob => "jdown_", // NB: Was "JGUP:"
Self::InferenceJob => "jinf_",
Self::MediaUpload => "mu_",
Self::User => "U:", // NB: Old-style prefix.
Self::VoiceConversionModel => "voco_",
}
}
}
@@ -59,4 +63,18 @@ mod tests {
assert!(entities.len() > 0);
assert_eq!(entities.len(), EntityType::COUNT);
}
#[test]
fn test_all_prefixes_end_with_separator() {
assert!(EntityType::iter()
.map(|entity| entity.prefix())
.all(|prefix| prefix.ends_with(":") || prefix.ends_with("_")));
}
#[test]
fn test_all_prefixes_end_with_separator_length_one() {
for prefix in EntityType::iter().map(|entity| entity.prefix()) {
assert_eq!(prefix.len() - 1, prefix.replace(":", "").replace("_", "").len());
}
}
}
crates/lib/tokens/src/users/mod.rs → crates/_types/tokens/src/users/mod.rs
View File
File renamed without changes.
crates/lib/tokens/src/users/user.rs → crates/_types/tokens/src/users/user.rs
-1
View File
@@ -4,7 +4,6 @@ use serde::Serialize;
use std::fmt::Debug;
/// The primary key for users.
// https://docs.rs/sqlx/latest/sqlx/trait.Type.html
#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
#[sqlx(transparent)]
pub struct UserToken(pub String);
crates/_types/tokens/src/voice_conversion/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod model;
crates/_types/tokens/src/voice_conversion/model.rs
+12
View File
@@ -0,0 +1,12 @@
use crate::prefixes::EntityType;
use serde::Deserialize;
use serde::Serialize;
use std::fmt::Debug;
/// The primary key for "generic" inference jobs.
#[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)]
#[sqlx(transparent)]
pub struct VoiceConversionModelToken(pub String);
impl_string_token!(VoiceConversionModelToken);
impl_crockford_generator!(VoiceConversionModelToken, 16usize, EntityType::VoiceConversionModel, CrockfordLower);
crates/_types/tokens/tests/media_upload_token_tests.rs
+166
View File
@@ -0,0 +1,166 @@
/// Test MediaUploadToken in isolation, just in case our macro-derived tests break.
use serde::Deserialize;
use serde::Serialize;
use tokens::files::media_upload::MediaUploadToken;
#[derive(Serialize, Deserialize, PartialEq, Debug)]
struct CompositeType {
media_upload_token: MediaUploadToken,
string: String,
}
mod interface {
use tokens::files::media_upload::MediaUploadToken;
#[test]
fn generate() {
let token = MediaUploadToken::generate();
assert!(!token.to_string().is_empty());
assert!(token.to_string().starts_with("mu_"));
}
#[test]
fn new() {
let token = MediaUploadToken::new("mu_foo".to_string());
assert_eq!(token, MediaUploadToken("mu_foo".to_string()));
}
#[test]
fn new_from_str() {
let token = MediaUploadToken::new_from_str("mu_foo");
assert_eq!(token, MediaUploadToken("mu_foo".to_string()));
}
#[test]
fn as_str() {
let token = MediaUploadToken("mu_foo".to_string());
assert_eq!(token.as_str(), "mu_foo");
}
#[test]
fn to_string() {
let token = MediaUploadToken("mu_foo".to_string());
assert_eq!(token.to_string(), "mu_foo".to_string());
}
}
mod traits {
use tokens::files::media_upload::MediaUploadToken;
#[test]
fn display() {
let token = MediaUploadToken("mu_foo".to_string());
assert_eq!(format!("{}", token), "mu_foo".to_string());
}
#[test]
fn debug() {
let token = MediaUploadToken("mu_foo".to_string());
assert_eq!(format!("{:?}", token), "MediaUploadToken(\"mu_foo\")".to_string());
}
}
mod serialization {
use crate::CompositeType;
use tokens::files::media_upload::MediaUploadToken;
#[test]
fn serialize() {
let expected = "\"mu_foo\"".to_string(); // NB: Quoted
let token = MediaUploadToken("mu_foo".to_string());
assert_eq!(expected, toml::to_string(&token).unwrap());
// Just to show this serializes the same as a string
assert_eq!(expected, toml::to_string("mu_foo").unwrap());
}
#[test]
fn nested_serialize() {
let value = CompositeType { media_upload_token: MediaUploadToken("mu_foo".to_string()), string: "bar".to_string() };
let expected = r#"{"media_upload_token":"mu_foo","string":"bar"}"#.to_string();
assert_eq!(expected, serde_json::to_string(&value).unwrap());
}
}
mod deserialization {
use crate::CompositeType;
use tokens::files::media_upload::MediaUploadToken;
#[test]
fn deserialize() {
let payload = "\"mu_foo\""; // NB: Quoted
let expected = "mu_foo".to_string();
let value: MediaUploadToken = serde_json::from_str(payload).unwrap();
assert_eq!(value, MediaUploadToken(expected.clone()));
// Just to show this deserializes the same way as a string
let value: String = serde_json::from_str(payload).unwrap();
assert_eq!(value, expected.clone());
}
#[test]
fn nested_deserialize() {
let payload = r#"{"media_upload_token":"mu_foo","string":"bar"}"#.to_string();
let expected = CompositeType {
media_upload_token: MediaUploadToken("mu_foo".to_string()),
string: "bar".to_string(),
};
assert_eq!(expected, serde_json::from_str::<CompositeType>(&payload).unwrap());
}
}
// These traits should be tested by the macro, but we duplicate them in case that breaks
mod crockford_traits {
use tokens::files::media_upload::MediaUploadToken;
const ENTROPIC_CHARACTERS_MINIMUM : usize = 8;
#[test]
fn entropy_is_sufficient() {
assert!(MediaUploadToken::entropic_character_len() > ENTROPIC_CHARACTERS_MINIMUM);
}
#[test]
fn token_length() {
assert_eq!(MediaUploadToken::generate().as_str().len(), 32);
}
#[test]
fn tokens_are_random() {
let mut tokens = std::collections::HashSet::new();
for _ in 0..100 {
tokens.insert(MediaUploadToken::generate().to_string());
}
assert_eq!(tokens.len(), 100);
}
#[test]
fn character_set() {
let token_string = MediaUploadToken::generate().to_string();
let prefix = MediaUploadToken::token_prefix();
let random_part = token_string.replace(prefix, "");
assert!(random_part.len() > ENTROPIC_CHARACTERS_MINIMUM);
assert!(random_part.chars().all(|c| c.is_numeric() || c.is_lowercase()));
}
#[test]
fn prefix_ends_with_separator() {
let prefix = MediaUploadToken::token_prefix();
assert!(prefix.ends_with(":") || prefix.ends_with("_"));
let token_string = MediaUploadToken::generate().to_string();
assert!(token_string.contains(":") || token_string.contains("_"));
}
#[test]
fn token_begins_with_prefix() {
let prefix = MediaUploadToken::token_prefix();
let token_string = MediaUploadToken::generate().to_string();
assert!(token_string.starts_with(prefix));
}
}
crates/lib/tokens/tests/user_token_tests.rs → crates/_types/tokens/tests/user_token_tests.rs
+3 -2
View File
@@ -17,6 +17,7 @@ mod interface {
fn generate() {
let token = UserToken::generate();
assert!(!token.to_string().is_empty());
assert!(token.to_string().starts_with("U:"));
}
#[test]
@@ -88,7 +89,7 @@ mod deserialization {
use tokens::users::user::UserToken;
#[test]
fn test_deserialize_trait() {
fn deserialize() {
let payload = "\"U:foo\""; // NB: Quoted
let expected = "U:foo".to_string();
@@ -101,7 +102,7 @@ mod deserialization {
}
#[test]
fn test_nested_deserialize_trait() {
fn nested_deserialize() {
let payload = r#"{"user_token":"U:foo","string":"bar"}"#.to_string();
let expected = CompositeType {
user_token: UserToken("U:foo".to_string()),
crates/lib/storage_buckets_common/Cargo.toml → crates/lib/cloud_storage/Cargo.toml
+3 -2
View File
@@ -1,5 +1,5 @@
[package]
name = "storage_buckets_common"
name = "cloud_storage"
edition = "2018"
version = "0.0.1"
authors = [
@@ -9,12 +9,13 @@ authors = [
publish = false
[lib]
name = "storage_buckets_common"
name = "cloud_storage"
path = "src/lib.rs"
[dependencies]
# Internal
buckets = { path = "../../_types/buckets" }
container_common = { path = "../container_common" }
# External
crates/lib/storage_buckets_common/src/bucket_client.rs → crates/lib/cloud_storage/src/bucket_client.rs
View File
File renamed without changes.
crates/lib/storage_buckets_common/src/bucket_path_unifier.rs → crates/lib/cloud_storage/src/bucket_path_unifier.rs
+35 -7
View File
@@ -1,9 +1,10 @@
use buckets::util::hashed_directory_path_short_string::hashed_directory_path_short_string;
use container_common::anyhow_result::AnyhowResult;
use crate::bucket_paths::hash_to_bucket_path;
use std::path::PathBuf;
/// This is designed to make it centrally configurable where
/// different types of objects are stored.
#[deprecated(note = "see the 'buckets' crate for a better approach")]
pub struct BucketPathUnifier {
// TTS
pub tts_synthesizer_model_root: PathBuf,
@@ -16,6 +17,8 @@ pub struct BucketPathUnifier {
pub w2l_inference_output_root: PathBuf,
pub w2l_model_root: PathBuf,
pub w2l_end_bump_root: PathBuf,
// VC
pub softvc_model_root: PathBuf,
}
impl BucketPathUnifier {
@@ -33,11 +36,14 @@ impl BucketPathUnifier {
w2l_inference_output_root: PathBuf::from("/w2l_inference_output"),
w2l_model_root: PathBuf::from("/w2l_pretrained_models"),
w2l_end_bump_root: PathBuf::from("/w2l_end_bumps"),
// VC
softvc_model_root: PathBuf::from("/user_uploaded_softvc_models"),
}
}
// ==================== TTS MODELS (SYNTHESIZER + VOCODER) ==================== //
// NB: Callers use "hash_file_sha2" as the file hash.
pub fn tts_synthesizer_path(&self, tts_synthesizer_file_hash: &str) -> PathBuf {
let hashed_path = Self::hashed_directory_path(tts_synthesizer_file_hash);
let model_filename = format!("{}.pt", &tts_synthesizer_file_hash);
@@ -65,7 +71,9 @@ impl BucketPathUnifier {
self.tts_pretrained_vocoder_model_root.join(tts_vocoder_model_name)
}
// NB: Callers use "hash_file_sha2" as the file hash.
/// User-uploaded vocoders.
/// These can be HifiGan and HifiGanSoftVc
pub fn vocoder_path(&self, vocoder_file_hash: &str) -> PathBuf {
let hashed_path = Self::hashed_directory_path(vocoder_file_hash);
let model_filename = format!("{}.pt", &vocoder_file_hash);
@@ -77,6 +85,7 @@ impl BucketPathUnifier {
// ==================== TTS INFERENCE OUTPUT ==================== //
// NB: Callers use job uuid_idempotency_token, which seems *bad*
/// This should include the string "vocodes" for downloaders.
pub fn tts_inference_wav_audio_output_path(&self, tts_inference_output_uuid: &str) -> PathBuf {
let hashed_path = Self::hashed_directory_path(tts_inference_output_uuid);
@@ -141,6 +150,7 @@ impl BucketPathUnifier {
// ==================== W2L INFERENCE OUTPUT ==================== //
// NB: caller uses inference job token, which seems ideal
// W2L inference output videos
pub fn w2l_inference_video_output_path(&self, w2l_inference_job_token: &str) -> PathBuf {
// NB: We don't want colons from the token in the filename.
@@ -168,13 +178,22 @@ impl BucketPathUnifier {
.join(video_filename)
}
// ==================== VOICE CONVERSION MODELS ==================== //
// NB: Callers use "hash_file_sha2" as the file hash.
pub fn softvc_model_path(&self, softvc_model_file_hash: &str) -> PathBuf {
let hashed_path = Self::hashed_directory_path(softvc_model_file_hash);
let model_filename = format!("{}.pt", &softvc_model_file_hash);
self.softvc_model_root
.join(hashed_path)
.join(model_filename)
}
// ==================== UTILITY ==================== //
pub fn hashed_directory_path(file_hash: &str) -> String {
match file_hash.len() {
0 | 1=> "".to_string(),
2 => format!("{}/", &file_hash[0..1]),
3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
_ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
}
hashed_directory_path_short_string(file_hash)
}
}
@@ -196,6 +215,8 @@ mod tests {
w2l_inference_output_root: PathBuf::from("/test_path_w2l_output"),
w2l_model_root: PathBuf::from("/test_path_w2l_pretrained_models"),
w2l_end_bump_root: PathBuf::from("/test_path_w2l_end_bumps"),
// VOICE CONVERSION
softvc_model_root: PathBuf::from("/test_path_softvc_models"),
}
}
@@ -296,6 +317,13 @@ mod tests {
"/test_path_w2l_output/a/b/c/vocodes_video_TYPEABCDEF.mp4");
}
#[test]
fn test_softvc_model_path() {
let paths = get_instance();
assert_eq!(paths.softvc_model_path("foobar").to_str().unwrap(),
"/test_path_softvc_models/f/o/o/foobar.pt");
}
#[test]
fn hashed_directory_path_length_zero() {
assert_eq!(&BucketPathUnifier::hashed_directory_path(""), "");
crates/lib/storage_buckets_common/src/lib.rs → crates/lib/cloud_storage/src/lib.rs
-1
View File
@@ -1,3 +1,2 @@
pub mod bucket_client;
pub mod bucket_path_unifier;
pub mod bucket_paths;
crates/lib/container_common/src/token/random_crockford_token.rs
+1
View File
@@ -3,6 +3,7 @@ use rand::Rng;
// Crockford characters
const CROCKFORD_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ";
#[deprecated(note = "see internal 'crockford' and 'token' crates")]
pub fn random_crockford_token(length: usize) -> String {
let mut rng = rand::thread_rng();
crates/lib/container_common/src/token/random_prefix_crockford_token.rs
+1
View File
@@ -10,6 +10,7 @@ const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
// Random part can't be any less than this.
const MIN_ENTROPY_LENGTH : usize = 8;
#[deprecated(note = "see internal 'crockford' and 'token' crates")]
pub fn random_prefix_crockford_token(prefix: &str, length: usize, uppercase: bool) -> AnyhowResult<String> {
let mut prefix = prefix.trim().to_string();
crates/lib/crockford/Cargo.toml
+21
View File
@@ -0,0 +1,21 @@
[package]
name = "crockford"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
# NB: Public "crockford" crate is non-sensically only for encoding and decoding u64s (wtf?)
# Our crate is for generating entropy and crockford encoding it.
name = "crockford"
path = "src/lib.rs"
[dependencies]
rand = "0.8.3"
[dev-dependencies]
# None yet
crates/lib/crockford/src/crockford_entropy.rs
+86
View File
@@ -0,0 +1,86 @@
use crate::{CROCKFORD_LOWERCASE_CHARSET, CROCKFORD_UPPERCASE_CHARSET};
use rand::Rng;
// TODO: Modify these routines to filter out intra-string swear words.
/// Generate a crockford-encoded entropy string (uppercase)
pub fn crockford_entropy_upper(length: usize) -> String {
crockford_entropy(length, CROCKFORD_UPPERCASE_CHARSET)
}
/// Generate a crockford-encoded entropy string (lowercase)
pub fn crockford_entropy_lower(length: usize) -> String {
crockford_entropy(length, CROCKFORD_LOWERCASE_CHARSET)
}
#[inline]
fn crockford_entropy(length: usize, character_set: &[u8]) -> String {
let mut rng = rand::thread_rng();
(0..length).map(|_| {
let idx = rng.gen_range(0..character_set.len());
character_set[idx] as char
}).collect()
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use crate::crockford_entropy::{crockford_entropy_lower, crockford_entropy_upper};
#[test]
fn crockford_entropy_upper_length() {
for len in 0..100 {
assert_eq!(crockford_entropy_upper(len).len(), len);
}
}
#[test]
fn crockford_entropy_lower_length() {
for len in 0..100 {
assert_eq!(crockford_entropy_lower(len).len(), len);
}
}
#[test]
fn crockford_entropy_upper_randomness() {
// NB: This *could* fail if the random pool or seed is bad and we generate duplicates.
let mut set = HashSet::new();
for _i in 0..100 {
let entropy = crockford_entropy_upper(16);
set.insert(entropy);
}
assert_eq!(set.len(), 100);
}
#[test]
fn crockford_entropy_lower_randomness() {
// NB: This *could* fail if the random pool or seed is bad and we generate duplicates.
let mut set = HashSet::new();
for _i in 0..100 {
let entropy = crockford_entropy_lower(16);
set.insert(entropy);
}
assert_eq!(set.len(), 100);
}
#[test]
fn crockford_entropy_upper_uses_uppercase() {
// NB: This *could* fail if the random pool or seed is bad and we get all numbers.
for _i in 0..100 {
let entropy = crockford_entropy_upper(16);
assert_ne!(entropy, entropy.to_lowercase());
assert_eq!(entropy, entropy.to_uppercase());
}
}
#[test]
fn crockford_entropy_upper_uses_lowercase() {
// NB: This *could* fail if the random pool or seed is bad and we get all numbers.
for _i in 0..100 {
let entropy = crockford_entropy_lower(16);
assert_ne!(entropy, entropy.to_uppercase());
assert_eq!(entropy, entropy.to_lowercase());
}
}
}
crates/lib/crockford/src/lib.rs
+31
View File
@@ -0,0 +1,31 @@
//! crockford
//!
//! The purpose of this library is to generate entropy and crockford encode it.
//!
// Never allow these
#![forbid(private_in_public)]
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
/// Crockford characters (uppercase)
const CROCKFORD_UPPERCASE_CHARSET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ";
/// Crockford characters (lowercase)
const CROCKFORD_LOWERCASE_CHARSET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
mod crockford_entropy;
// Re-export
pub use crockford_entropy::crockford_entropy_upper;
pub use crockford_entropy::crockford_entropy_lower;
crates/lib/database_queries/Cargo.toml
+5 -2
View File
@@ -15,10 +15,13 @@ path = "src/lib.rs"
[dependencies]
# Internal
buckets = { path = "../../_types/buckets" }
config = { path = "../config" }
container_common = { path = "../container_common" }
reusable_types = { path = "../reusable_types" }
tokens = { path = "../tokens" }
enums = { path = "../../_types/enums" }
errors = { path = "../errors" }
reusable_types = { path = "../../_types/reusable_types" }
tokens = { path = "../../_types/tokens" }
# External
anyhow = "1.0.40"
crates/lib/database_queries/src/column_types/mod.rs
-1
View File
@@ -1,5 +1,4 @@
pub mod job_status;
pub mod record_visibility;
pub mod tts_model_type;
pub mod twitch_event_category;
pub mod vocoder_type;
crates/lib/database_queries/src/column_types/record_visibility.rs
-39
View File
@@ -1,39 +0,0 @@
//! These are columns where users can control the visibility of their data.
use anyhow::anyhow;
use container_common::anyhow_result::AnyhowResult;
/// To use this in a query, the query must have type annotations.
/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375
/// eg. preferred_tts_result_visibility as `preferred_tts_result_visibility: crate::column_types::record_visibility::RecordVisibility`
#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)]
#[serde(rename_all = "lowercase")]
#[sqlx(rename_all = "lowercase")]
pub enum RecordVisibility {
Public,
Hidden,
Private,
}
impl Default for RecordVisibility {
fn default() -> Self { Self::Public }
}
impl RecordVisibility {
pub fn to_str(&self) -> &'static str {
match self {
RecordVisibility::Public => "public",
RecordVisibility::Hidden => "hidden",
RecordVisibility::Private => "private",
}
}
pub fn from_str(record_visibility: &str) -> AnyhowResult<Self> {
match record_visibility {
"public" => Ok(RecordVisibility::Public),
"hidden" => Ok(RecordVisibility::Hidden),
"private" => Ok(RecordVisibility::Private),
_ => Err(anyhow!("invalid value: {:?}", record_visibility)),
}
}
}
crates/lib/database_queries/src/lib.rs
+1
View File
@@ -6,5 +6,6 @@ pub mod column_types;
pub mod complex_models;
pub mod helpers;
pub mod mediators;
pub mod payloads;
pub mod queries;
pub mod tokens;
crates/lib/database_queries/src/mediators/badge_granter.rs
+84 -2
View File
@@ -38,7 +38,11 @@ impl BadgeGranter {
let mut maybe_badge = None;
if count >= 250 {
if count >= 1000 {
maybe_badge = Some(UserBadge::TtsModelUploader1000);
} else if count >= 500 {
maybe_badge = Some(UserBadge::TtsModelUploader500);
} else if count >= 250 {
maybe_badge = Some(UserBadge::TtsModelUploader250);
} else if count >= 200 {
maybe_badge = Some(UserBadge::TtsModelUploader200);
@@ -133,7 +137,11 @@ impl BadgeGranter {
let mut maybe_badge = None;
if count >= 250 {
if count >= 1000 {
maybe_badge = Some(UserBadge::VocoderModelUploader1000);
} else if count >= 500 {
maybe_badge = Some(UserBadge::VocoderModelUploader500);
} else if count >= 250 {
maybe_badge = Some(UserBadge::VocoderModelUploader250);
} else if count >= 200 {
maybe_badge = Some(UserBadge::VocoderModelUploader200);
@@ -173,6 +181,56 @@ impl BadgeGranter {
Ok(())
}
/// This needs to be called *after* successful upload.
pub async fn maybe_grant_softvc_vocoder_model_uploads_badge(&self, user_token: &str) -> AnyhowResult<()> {
let count = self.count_softvc_vocoder_models_uploaded(user_token).await?;
let mut maybe_badge = None;
if count >= 1000 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader1000);
} else if count >= 500 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader500);
} else if count >= 250 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader250);
} else if count >= 200 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader200);
} else if count >= 150 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader150);
} else if count >= 100 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader100);
} else if count >= 50 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader50);
} else if count >= 20 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader20);
} else if count >= 10 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader10);
} else if count >= 5 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader5);
} else if count >= 1 {
maybe_badge = Some(UserBadge::VocoderRocketVcModelUploader1);
}
let badge = match maybe_badge {
Some(badge) => badge,
None => return Ok(()),
};
if self.has_badge(user_token, badge).await? {
return Ok(())
}
let _record_id = self.insert(
badge,
user_token,
).await?;
self.firehose_publisher.publish_user_badge_granted(user_token, badge.to_db_value())
.await?;
Ok(())
}
// =======================================================================
pub async fn has_badge(&self, user_token: &str, user_badge: UserBadge) -> AnyhowResult<bool> {
@@ -245,6 +303,30 @@ SELECT count(*) as count
FROM vocoder_models
WHERE
creator_user_token = ?
AND vocoder_type = 'hifigan'
LIMIT 1
"#,
user_token
)
.fetch_one(&self.mysql_pool)
.await;
self.handle_count_query(maybe_result)
}
async fn count_softvc_vocoder_models_uploaded(
&self,
user_token: &str,
) -> AnyhowResult<u64> {
// NB: This could get expensive!
let maybe_result = sqlx::query_as!(
CountRecord,
r#"
SELECT count(*) as count
FROM vocoder_models
WHERE
creator_user_token = ?
AND vocoder_type = 'hifigan_rocket_vc'
LIMIT 1
"#,
user_token
crates/lib/database_queries/src/mediators/firehose_publisher.rs
+51 -3
View File
@@ -6,12 +6,16 @@ use sqlx::error::Error::Database;
use sqlx::mysql::MySqlQueryResult;
use sqlx::{MySqlPool};
use std::sync::Arc;
use tokens::files::media_upload::MediaUploadToken;
use tokens::jobs::inference::InferenceJobToken;
use tokens::users::user::UserToken;
// TODO(bt, 2022-12-19): Convert this to a database 'enum'. Also, create an 'enums' package similar to 'tokens'.
#[derive(Debug, Clone, Copy)]
enum FirehoseEvent {
UserSignUp,
// We don't publish for all badges (eg. early user signup)
// NB: We don't publish for all badges (eg. early user signup)
UserBadgeGranted,
TtsModelUploadStarted,
@@ -24,9 +28,15 @@ enum FirehoseEvent {
W2lInferenceStarted,
W2lInferenceCompleted,
VcInferenceStarted,
VcInferenceCompleted,
GenericDownloadStarted,
GenericDownloadCompleted,
MediaUploaded,
// TODO(bt, 2022-12-19): Are the following unused, merely planned (?)
TwitterMention,
TwitterRetweet,
DiscordJoin,
@@ -48,14 +58,19 @@ impl FirehoseEvent {
FirehoseEvent::W2lTemplateUploadCompleted => "w2l_template_upload_completed",
FirehoseEvent::W2lInferenceStarted => "w2l_inference_started",
FirehoseEvent::W2lInferenceCompleted => "w2l_inference_completed",
FirehoseEvent::VcInferenceStarted => "vc_inference_started",
FirehoseEvent::VcInferenceCompleted => "vc_inference_completed",
FirehoseEvent::GenericDownloadStarted => "generic_download_started",
FirehoseEvent::GenericDownloadCompleted => "generic_download_completed",
FirehoseEvent::MediaUploaded => "media_uploaded",
// Are the following unused (?)
FirehoseEvent::TwitterMention => "twitter_mention",
FirehoseEvent::TwitterRetweet => "twitter_retweet",
FirehoseEvent::DiscordJoin => "discord_join",
FirehoseEvent::DiscordMessage => "discord_message",
FirehoseEvent::TwitchSubscribe => "twitch_subscribe",
FirehoseEvent::TwitchFollow => "twitch_follow",
FirehoseEvent::GenericDownloadStarted => "generic_download_started",
FirehoseEvent::GenericDownloadCompleted => "generic_download_completed",
}
}
}
@@ -177,6 +192,27 @@ impl FirehosePublisher {
Ok(())
}
pub async fn enqueue_vc_inference(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken) -> AnyhowResult<()> {
let _record_id = self.insert(
FirehoseEvent::VcInferenceStarted,
maybe_user_token.map(|u| u.as_str()),
Some(inference_job_token.as_str()),
Some(inference_job_token.as_str()),
).await?;
Ok(())
}
// TODO: Change result token type.
pub async fn vc_inference_finished(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken, result_token: &str) -> AnyhowResult<()> {
let _record_id = self.insert(
FirehoseEvent::VcInferenceCompleted,
maybe_user_token.map(|u| u.as_str()),
Some(inference_job_token.as_str()), // TODO: This could be vc model token
Some(result_token)
).await?;
Ok(())
}
pub async fn enqueue_generic_download(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> {
let _record_id = self.insert(
FirehoseEvent::GenericDownloadStarted,
@@ -198,6 +234,16 @@ impl FirehosePublisher {
Ok(())
}
pub async fn publish_media_uploaded(&self, maybe_user_token: Option<&UserToken>, upload_token: &MediaUploadToken) -> AnyhowResult<()> {
let _record_id = self.insert(
FirehoseEvent::MediaUploaded,
maybe_user_token.map(|u| u.as_str()),
Some(upload_token.as_str()),
Some(upload_token.as_str()),
).await?;
Ok(())
}
// =======================================================================
async fn insert(
@@ -237,6 +283,8 @@ SET
Ok(res) => {
res.last_insert_id()
},
// TODO(bt, 2022-12-20): I've never richly handled database errors/error codes.
// I should revisit these in the future and consider some kind of middleware.
Err(err) => {
warn!("Insert record DB error: {:?}", err);
crates/lib/database_queries/src/payloads/generic_inference_args.rs
+86
View File
@@ -0,0 +1,86 @@
use enums::workers::generic_inference_type::GenericInferenceType;
use tokens::files::media_upload::MediaUploadToken;
use tokens::voice_conversion::model::VoiceConversionModelToken;
/// Used to encode extra state for the `generic_inference_jobs` table.
/// This should act somewhat like a serialized protobuf stored inside a record.
#[derive(Clone, Serialize, Deserialize)]
pub struct GenericInferenceArgs {
/// The type of inference (probably also present in a top-level field)
pub inference_type: Option<GenericInferenceType>,
/// REQUIRED.
/// Actual type-specific arguments.
pub args: Option<PolymorphicInferenceArgs>,
}
#[derive(Clone, Serialize, Deserialize)]
pub enum PolymorphicInferenceArgs {
TextToSpeechInferenceArgs {
// TODO
},
VoiceConversionInferenceArgs {
/// REQUIRED.
/// The voice conversion model to use.
model_token: Option<VoiceConversionModelToken>,
/// OPTIONAL.
/// If set, the media file to use as the source in the conversion.
/// It's "optional" in case we use other types of records in the future.
maybe_media_token: Option<MediaUploadToken>,
},
}
#[cfg(test)]
mod tests {
use crate::payloads::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
use enums::workers::generic_inference_type::GenericInferenceType;
use tokens::files::media_upload::MediaUploadToken;
use tokens::voice_conversion::model::VoiceConversionModelToken;
#[test]
fn typical_voice_conversion_args_serialize() {
let args = GenericInferenceArgs {
inference_type: Some(GenericInferenceType::VoiceConversion),
args: Some(PolymorphicInferenceArgs::VoiceConversionInferenceArgs {
model_token: Some(VoiceConversionModelToken::new_from_str("vc_model_token")),
maybe_media_token: Some(MediaUploadToken::new_from_str("media_token")),
}),
};
let json = serde_json::ser::to_string(&args).unwrap();
// NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating.
assert_eq!(json,
r#"{"inference_type":"voice_conversion","args":{"VoiceConversionInferenceArgs":{"model_token":"vc_model_token","maybe_media_token":"media_token"}}}"#.to_string());
// NB: Make sure we don't overflow the DB field capacity (TEXT column).
assert!(json.len() < 1000);
}
#[test]
fn serialize_nullable_form() {
let mut args : Option<GenericInferenceArgs> = None;
let json = serde_json::ser::to_string(&args).unwrap();
// NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating.
assert_eq!(json, "null");
args = Some(GenericInferenceArgs {
inference_type: Some(GenericInferenceType::VoiceConversion),
args: Some(PolymorphicInferenceArgs::VoiceConversionInferenceArgs {
model_token: Some(VoiceConversionModelToken::new_from_str("vc_model_token")),
maybe_media_token: Some(MediaUploadToken::new_from_str("media_token")),
}),
});
let json = serde_json::ser::to_string(&args).unwrap();
// NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating.
assert_eq!(json,
r#"{"inference_type":"voice_conversion","args":{"VoiceConversionInferenceArgs":{"model_token":"vc_model_token","maybe_media_token":"media_token"}}}"#.to_string());
// NB: Make sure we don't overflow the DB field capacity (TEXT column).
assert!(json.len() < 1000);
}
}
crates/lib/database_queries/src/payloads/media_upload_details.rs
+4
View File
@@ -0,0 +1,4 @@
pub struct MediaUploadDetails {
// TODO
}
crates/lib/database_queries/src/payloads/mod.rs
+4
View File
@@ -0,0 +1,4 @@
//! This module is used to serialize values to database 'TEXT' fields.
pub mod generic_inference_args;
pub mod media_upload_details;
crates/lib/database_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs
+6 -6
View File
@@ -4,8 +4,8 @@ use container_common::anyhow_result::AnyhowResult;
use crate::column_types::job_status::JobStatus;
use crate::queries::generic_download::job::_keys::GenericDownloadJobId;
use crate::tokens::Tokens;
use reusable_types::entity_visibility::EntityVisibility;
use reusable_types::generic_download_type::GenericDownloadType;
use enums::core::visibility::Visibility;
use enums::workers::generic_download_type::GenericDownloadType;
use sqlx::MySqlPool;
use std::path::Path;
use tokens::jobs::download::DownloadJobToken;
@@ -18,7 +18,7 @@ pub struct AvailableDownloadJob {
pub creator_user_token: String,
pub creator_ip_address: String,
pub creator_set_visibility: EntityVisibility,
pub creator_set_visibility: Visibility,
pub download_type: GenericDownloadType,
pub download_url: String,
@@ -45,9 +45,9 @@ SELECT
creator_user_token,
creator_ip_address,
creator_set_visibility as `creator_set_visibility: reusable_types::entity_visibility::EntityVisibility`,
creator_set_visibility as `creator_set_visibility: enums::core::visibility::Visibility`,
download_type as `download_type: reusable_types::generic_download_type::GenericDownloadType`,
download_type as `download_type: enums::workers::generic_download_type::GenericDownloadType`,
download_url,
title,
@@ -108,7 +108,7 @@ struct AvailableDownloadJobRawInternal {
pub creator_user_token: String,
pub creator_ip_address: String,
pub creator_set_visibility: EntityVisibility,
pub creator_set_visibility: Visibility,
pub download_type: GenericDownloadType,
pub download_url: String,
crates/lib/database_queries/src/queries/generic_download/mod.rs
+1 -2
View File
@@ -1,3 +1,2 @@
pub mod get_generic_download_job_status;
pub mod insert_generic_download_job;
pub mod job;
pub mod web;
crates/lib/database_queries/src/queries/generic_download/get_generic_download_job_status.rs → crates/lib/database_queries/src/queries/generic_download/web/get_generic_download_job_status.rs
-2
View File
@@ -2,8 +2,6 @@ use anyhow::anyhow;
use chrono::{DateTime, Utc};
use container_common::anyhow_result::AnyhowResult;
use log::warn;
use reusable_types::entity_visibility::EntityVisibility;
use reusable_types::generic_download_type::GenericDownloadType;
use sqlx::MySqlPool;
use tokens::jobs::download::DownloadJobToken;
crates/lib/database_queries/src/queries/generic_download/insert_generic_download_job.rs → crates/lib/database_queries/src/queries/generic_download/web/insert_generic_download_job.rs
+3 -3
View File
@@ -1,7 +1,7 @@
use anyhow::anyhow;
use container_common::anyhow_result::AnyhowResult;
use reusable_types::entity_visibility::EntityVisibility;
use reusable_types::generic_download_type::GenericDownloadType;
use enums::core::visibility::Visibility;
use enums::workers::generic_download_type::GenericDownloadType;
use sqlx::MySqlPool;
use tokens::jobs::download::DownloadJobToken;
@@ -13,7 +13,7 @@ pub struct Args <'a> {
pub title: &'a str,
pub creator_user_token: &'a str,
pub creator_ip_address: &'a str,
pub creator_set_visibility: EntityVisibility,
pub creator_set_visibility: Visibility,
pub mysql_pool: &'a MySqlPool,
}
crates/lib/database_queries/src/queries/generic_download/web/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod get_generic_download_job_status;
pub mod insert_generic_download_job;
Loaded 100 of 476 files, more files were not shown because too many files have changed in this diff. Show more