From 84c616b6f619cb89d4bcd8918f977e86693585bb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 29 Aug 2022 00:28:26 -0400 Subject: [PATCH] New lib and service crates --- Cargo.lock | 139 ++++++++++++++++++ Cargo.toml | 4 + crates/lib/http_endpoints_common/README.md | 5 + .../http_endpoints_common/billing/Cargo.toml | 38 +++++ .../http_endpoints_common/billing/src/lib.rs | 0 .../inference_workloads/Cargo.toml | 38 +++++ .../inference_workloads/src/lib.rs | 0 .../http_endpoints_common/users/Cargo.toml | 38 +++++ .../http_endpoints_common/users/src/lib.rs | 0 .../image_generation_service/Cargo.toml | 126 ++++++++++++++++ .../image_generation_service/src/main.rs | 0 .../image_generation_service/src/routes.rs | 0 12 files changed, 388 insertions(+) create mode 100644 crates/lib/http_endpoints_common/README.md create mode 100644 crates/lib/http_endpoints_common/billing/Cargo.toml create mode 100644 crates/lib/http_endpoints_common/billing/src/lib.rs create mode 100644 crates/lib/http_endpoints_common/inference_workloads/Cargo.toml create mode 100644 crates/lib/http_endpoints_common/inference_workloads/src/lib.rs create mode 100644 crates/lib/http_endpoints_common/users/Cargo.toml create mode 100644 crates/lib/http_endpoints_common/users/src/lib.rs create mode 100644 crates/service/image_generation_service/Cargo.toml create mode 100644 crates/service/image_generation_service/src/main.rs create mode 100644 crates/service/image_generation_service/src/routes.rs diff --git a/Cargo.lock b/Cargo.lock index 38ffd292c4..41fd804dd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1933,6 +1933,69 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eee9694f83d9b7c09682fdb32213682939507884e5bcf227be9aff5d644b90dc" +[[package]] +name = "http_endpoints_common_billing" +version = "0.0.1" +dependencies = [ + "actix-cors 0.6.0-beta.3", + "actix-http 3.0.0-beta.11", + "actix-web 4.0.0-beta.10", + "anyhow", + "base64 0.13.0", + "chrono", + "container_common", + "dotenv", + "log", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "toml", +] + +[[package]] +name = "http_endpoints_common_inference_workloads" +version = "0.0.1" +dependencies = [ + "actix-cors 0.6.0-beta.3", + "actix-http 3.0.0-beta.11", + "actix-web 4.0.0-beta.10", + "anyhow", + "base64 0.13.0", + "chrono", + "container_common", + "dotenv", + "log", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "toml", +] + +[[package]] +name = "http_endpoints_common_users" +version = "0.0.1" +dependencies = [ + "actix-cors 0.6.0-beta.3", + "actix-http 3.0.0-beta.11", + "actix-web 4.0.0-beta.10", + "anyhow", + "base64 0.13.0", + "chrono", + "container_common", + "dotenv", + "log", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "toml", +] + [[package]] name = "http_server_common" version = "0.0.1" @@ -2065,6 +2128,82 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image-generation-service" +version = "0.0.1" +dependencies = [ + "actix", + "actix-cors 0.6.0-beta.3", + "actix-files", + "actix-http 3.0.0-beta.11", + "actix-multipart", + "actix-rt 2.4.0", + "actix-service 2.0.1", + "actix-utils 3.0.0", + "actix-web 4.0.0-beta.10", + "actix-web-actors", + "anyhow", + "base64 0.13.0", + "bcrypt", + "chrono", + "config", + "container_common", + "data-encoding", + "database_queries", + "derive_more", + "diesel", + "dotenv", + "easyenv", + "futures 0.1.31", + "futures 0.3.17", + "futures-util", + "google_drive_common", + "hex", + "hmac 0.10.1", + "hostname", + "http_server_common", + "hyper 0.14.14", + "infer", + "jobs_common", + "jwt", + "language-tags 0.3.2", + "lazy_static", + "lexical-sort", + "lfu", + "limitation", + "log", + "lru_time_cache", + "magic-crypt", + "md-5", + "memory_caching", + "native-tls", + "once_cell", + "pulldown-cmark", + "r2d2_redis", + "rand 0.8.4", + "redis_common", + "regex", + "reqwest", + "reusable_types", + "ring 0.17.0-alpha.10", + "rust-s3", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sqlx", + "storage_buckets_common", + "subprocess", + "tempdir", + "time 0.2.26", + "tokio 1.13.0", + "toml", + "tts_common", + "twitch_common", + "user_input_common", + "uuid", +] + [[package]] name = "indexmap" version = "1.6.2" diff --git a/Cargo.toml b/Cargo.toml index 64686d9906..bba440c387 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,9 @@ members = [ "crates/lib/container_common", "crates/lib/database_queries", "crates/lib/google_drive_common", + "crates/lib/http_endpoints_common/billing", + "crates/lib/http_endpoints_common/inference_workloads", + "crates/lib/http_endpoints_common/users", "crates/lib/http_server_common", "crates/lib/jobs_common", "crates/lib/memory_caching", @@ -18,6 +21,7 @@ members = [ # Application "crates/service/generic_download_job", + "crates/service/image_generation_service", "crates/service/storyteller_web", "crates/service/tts_download_job", "crates/service/tts_inference_job", diff --git a/crates/lib/http_endpoints_common/README.md b/crates/lib/http_endpoints_common/README.md new file mode 100644 index 0000000000..75efd4eae1 --- /dev/null +++ b/crates/lib/http_endpoints_common/README.md @@ -0,0 +1,5 @@ +http_endpoints_common +===================== +This directory contains Rust library packages that contain common and reusable HTTP endpoints. +These might be things that are incorporated into different services, but otherwise broadly reusable. +User account creation, login, billing, etc. diff --git a/crates/lib/http_endpoints_common/billing/Cargo.toml b/crates/lib/http_endpoints_common/billing/Cargo.toml new file mode 100644 index 0000000000..d19acffe74 --- /dev/null +++ b/crates/lib/http_endpoints_common/billing/Cargo.toml @@ -0,0 +1,38 @@ +[package] +publish = false +name = "http_endpoints_common_billing" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +[lib] +name = "http_endpoints_common_billing" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../../container_common" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +dotenv = "0.15.0" +log = "0.4.14" +once_cell = "1.9.0" +regex = "1.5.4" + +# Web +actix-cors = "0.6.0-beta.3" +actix-http = "3.0.0-beta.11" +actix-web = "4.0.0-beta.10" + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" diff --git a/crates/lib/http_endpoints_common/billing/src/lib.rs b/crates/lib/http_endpoints_common/billing/src/lib.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/lib/http_endpoints_common/inference_workloads/Cargo.toml b/crates/lib/http_endpoints_common/inference_workloads/Cargo.toml new file mode 100644 index 0000000000..397227d93b --- /dev/null +++ b/crates/lib/http_endpoints_common/inference_workloads/Cargo.toml @@ -0,0 +1,38 @@ +[package] +publish = false +name = "http_endpoints_common_inference_workloads" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +[lib] +name = "http_endpoints_common_inference_workloads" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../../container_common" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +dotenv = "0.15.0" +log = "0.4.14" +once_cell = "1.9.0" +regex = "1.5.4" + +# Web +actix-cors = "0.6.0-beta.3" +actix-http = "3.0.0-beta.11" +actix-web = "4.0.0-beta.10" + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" diff --git a/crates/lib/http_endpoints_common/inference_workloads/src/lib.rs b/crates/lib/http_endpoints_common/inference_workloads/src/lib.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/lib/http_endpoints_common/users/Cargo.toml b/crates/lib/http_endpoints_common/users/Cargo.toml new file mode 100644 index 0000000000..6ab8a4f8ad --- /dev/null +++ b/crates/lib/http_endpoints_common/users/Cargo.toml @@ -0,0 +1,38 @@ +[package] +publish = false +name = "http_endpoints_common_users" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +[lib] +name = "http_endpoints_common_users" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../../container_common" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +dotenv = "0.15.0" +log = "0.4.14" +once_cell = "1.9.0" +regex = "1.5.4" + +# Web +actix-cors = "0.6.0-beta.3" +actix-http = "3.0.0-beta.11" +actix-web = "4.0.0-beta.10" + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" diff --git a/crates/lib/http_endpoints_common/users/src/lib.rs b/crates/lib/http_endpoints_common/users/src/lib.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/image_generation_service/Cargo.toml b/crates/service/image_generation_service/Cargo.toml new file mode 100644 index 0000000000..90948851b9 --- /dev/null +++ b/crates/service/image_generation_service/Cargo.toml @@ -0,0 +1,126 @@ +[package] +publish = false +name = "image-generation-service" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] + +# Main server monolith +[[bin]] +name = "image-generation-service" +path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } +database_queries = { path = "../../lib/database_queries" } +google_drive_common = { path = "../../lib/google_drive_common" } +http_server_common = { path = "../../lib/http_server_common" } +jobs_common = { path = "../../lib/jobs_common" } +memory_caching = { path = "../../lib/memory_caching" } +redis_common = { path = "../../lib/redis_common" } +reusable_types = { path = "../../lib/reusable_types" } +storage_buckets_common = { path = "../../lib/storage_buckets_common" } +tts_common = { path = "../../lib/tts_common" } +twitch_common = { path = "../../lib/twitch_common" } +user_input_common = { path = "../../lib/user_input_common" } + +# External +anyhow = "1.0.40" +base64 = "0.13.0" +chrono = { version = "0.4.19", features = ["serde"] } +derive_more = "0.99.14" +dotenv = "0.15.0" +easyenv = "0.0.6" +hex = "0.4.3" +hostname = "0.3.1" +lazy_static = "1.4.0" +lexical-sort = "0.3.1" +lfu = "0.2.5" +log = "0.4.14" +once_cell = "1.9.0" +rand = "0.8.3" +regex = "1.5.4" +time = "0.2.26" +uuid = { version = "0.8.2", features = ["v4"] } + +# Language tag parsing +# TODO: Move to language support crate +language-tags = "0.3.2" # Parsing single tags richly + +# HTTP +# TODO/NB: This is just for Twitch, but we might need it later. +reqwest = { version = "0.11.4" } +native-tls = "0.2.8" + +# Runtime +# NB: We switched to Tokio from `async-std` for broader ecosystem support. +tokio = { version = "1.6.0", features = ["macros"] } + +# Futures +futures = { version = "0.3.17", features = ["thread-pool"] } # feature for ThreadPool +futures-util = "0.3.9" +futures_old_for_limiter = { package = "futures", version = "0.1.29" } # Older version of futures for `limiter` crate + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } + +# Redis Database +limitation = "0.1.1" # Redis rate limiter +r2d2_redis = "0.14.0" + +# In-memory caching +lru_time_cache = "0.11.11" + +# Files +data-encoding = "2.3.2" +ring = "0.17.0-alpha.10" +rust-s3 = { version = "0.28", features = ["never-encode-slash"] } +infer = "0.5.0" + +subprocess = "0.2.6" +tempdir = "0.3.7" + +actix = "0.12.0" +actix-cors = "0.6.0-beta.3" +actix-files= "0.6.0-beta.8" +actix-http = "3.0.0-beta.11" +actix-multipart = "0.4.0-beta.7" +actix-rt = "2.4.0" +actix-service = "2.0.1" +actix-utils = "3.0.0" +actix-web = "4.0.0-beta.10" +actix-web-actors = "4.0.0-beta.7" +jwt = "0.13.0" + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.14.14" } + +# Crypto +bcrypt = "0.8" # password hashes +hmac = "0.10.0" # cookies +magic-crypt = "3.1.8" # encrypting sort IDs for the frontend so we don't leak info +md-5 = "0.9.1" # gravatar +sha2 = "0.9.5" # file hashes, inference text hashes, etc. + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" +toml = "0.5.8" + +# Text processing +pulldown-cmark = "0.8.0" + +[build-dependencies] +# NB: Only using diesel for migrations support +diesel = { version = "1.4.6", features = ["mysql"] } diff --git a/crates/service/image_generation_service/src/main.rs b/crates/service/image_generation_service/src/main.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/image_generation_service/src/routes.rs b/crates/service/image_generation_service/src/routes.rs new file mode 100644 index 0000000000..e69de29bb2