From bc1ccbc1769c2e3a0c471c53dd36b9bfdde91dec Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 02:36:10 -0400 Subject: [PATCH] job can query results! --- Cargo.lock | 565 +++++++++++++++++++++++++++++++++++++++- Cargo.toml | 9 + src/tts_download_job.rs | 128 ++++++++- 3 files changed, 697 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fa0885f1c..0a85409874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,7 @@ dependencies = [ "log", "pin-project 0.4.28", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -97,7 +97,7 @@ dependencies = [ "futures-core", "futures-util", "fxhash", - "h2", + "h2 0.2.7", "http", "httparse", "indexmap", @@ -372,6 +372,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "async-channel" version = "1.6.1" @@ -486,6 +496,7 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" dependencies = [ + "async-attributes", "async-channel", "async-global-executor", "async-io", @@ -540,6 +551,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" +[[package]] +name = "attohttpc" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb8867f378f33f78a811a8eb9bf108ad99430d7aad43315dd9319c827ef6247" +dependencies = [ + "http", + "log", + "native-tls", + "openssl", + "serde", + "serde_json", + "url 2.2.2", + "wildmatch", +] + [[package]] name = "atty" version = "0.2.14" @@ -587,6 +614,31 @@ dependencies = [ "serde_urlencoded", ] +[[package]] +name = "aws-creds" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1331d069460a674d42bd27c12b47ce578f789954c7bd7f239fd030771eca6616" +dependencies = [ + "anyhow", + "attohttpc", + "dirs", + "rust-ini", + "serde", + "serde-xml-rs", + "serde_derive", + "url 2.2.2", +] + +[[package]] +name = "aws-region" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2884b8f2aaeb4a4bf80b219b4fe1d340139ca9331679c57e0fd4a24f571a78bd" +dependencies = [ + "anyhow", +] + [[package]] name = "base-x" version = "0.2.8" @@ -906,6 +958,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + [[package]] name = "ctor" version = "0.1.20" @@ -960,12 +1021,41 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + [[package]] name = "discard" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dlv-list" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b" +dependencies = [ + "rand 0.8.3", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -1269,6 +1359,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "google-drive" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fe473f3853a7668300abb646ffb01b6eb7b97548f24575f73079b03e5c73b2" +dependencies = [ + "bytes 1.0.1", + "reqwest", + "serde", + "yup-oauth2", +] + [[package]] name = "h2" version = "0.2.7" @@ -1284,11 +1386,30 @@ dependencies = [ "indexmap", "slab", "tokio 0.2.25", - "tokio-util", + "tokio-util 0.3.1", "tracing", "tracing-futures", ] +[[package]] +name = "h2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726" +dependencies = [ + "bytes 1.0.1", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio 1.5.0", + "tokio-util 0.6.7", + "tracing", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -1363,12 +1484,29 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +dependencies = [ + "bytes 1.0.1", + "http", + "pin-project-lite 0.2.6", +] + [[package]] name = "httparse" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" +[[package]] +name = "httpdate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" + [[package]] name = "humantime" version = "1.3.0" @@ -1378,6 +1516,60 @@ dependencies = [ "quick-error", ] +[[package]] +name = "hyper" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" +dependencies = [ + "bytes 1.0.1", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.3", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 1.0.7", + "socket2 0.4.0", + "tokio 1.5.0", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio 1.5.0", + "tokio-rustls", + "webpki", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper", + "native-tls", + "tokio 1.5.0", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.1.5" @@ -1437,9 +1629,15 @@ dependencies = [ "socket2 0.3.19", "widestring", "winapi 0.3.9", - "winreg", + "winreg 0.6.2", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itoa" version = "0.4.7" @@ -1587,6 +1785,34 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-async" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3919783e441e38b0512fae9eebd87c445c42e39508231fba2a8b9fd7ba88e999" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer", + "digest", + "opaque-debug", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.4.0" @@ -1609,6 +1835,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minidom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332592c2149fc7dd40a64fc9ef6f0d65607284b474cef9817d1fc8c7e7b3608e" +dependencies = [ + "quick-xml", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1880,6 +2115,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485" +dependencies = [ + "dlv-list", + "hashbrown", +] + [[package]] name = "parking" version = "2.0.0" @@ -2044,6 +2289,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.9" @@ -2149,6 +2403,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom 0.2.2", + "redox_syscall", +] + [[package]] name = "regex" version = "1.5.4" @@ -2175,6 +2439,41 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "reqwest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +dependencies = [ + "base64 0.13.0", + "bytes 1.0.1", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite 0.2.6", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 1.5.0", + "tokio-native-tls", + "url 2.2.2", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.7.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -2185,6 +2484,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rsa" version = "0.4.0" @@ -2205,6 +2519,48 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-ini" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b134767a87e0b086f73a4ce569ac9ce7d202f39c8eab6caa266e2617e73ac6" +dependencies = [ + "cfg-if 0.1.10", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.27.0-rc4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c93272c1d654d492f8ab30b94cd43d98f2700b1db55b2576aff7712ce40e3ef" +dependencies = [ + "anyhow", + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.0", + "cfg-if 1.0.0", + "chrono", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding 2.1.0", + "reqwest", + "serde", + "serde-xml-rs", + "serde_derive", + "sha2", + "tokio 1.5.0", + "tokio-stream", + "url 2.2.2", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -2214,6 +2570,31 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "ryu" version = "1.0.5" @@ -2236,6 +2617,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.2.0" @@ -2283,6 +2680,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-xml-rs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa" +dependencies = [ + "log", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.125" @@ -2599,6 +3008,7 @@ dependencies = [ "actix-utils 3.0.0", "actix-web", "anyhow", + "async-std", "base64 0.13.0", "bcrypt", "chrono", @@ -2606,13 +3016,17 @@ dependencies = [ "diesel", "easyenv", "futures", + "google-drive", + "hex", "hmac", "hostname", "jwt", "lazy_static", "log", + "md-5", "rand 0.8.3", "regex", + "rust-s3", "serde", "serde_derive", "serde_json", @@ -2810,15 +3224,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" dependencies = [ "autocfg 1.0.1", + "bytes 1.0.1", "libc", + "memchr", "mio 0.7.11", "once_cell", "parking_lot", "pin-project-lite 0.2.6", "signal-hook-registry", + "tokio-macros", "winapi 0.3.9", ] +[[package]] +name = "tokio-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio 1.5.0", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio 1.5.0", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8864d706fdb3cc0843a49647ac892720dac98a6eeb818b77190592cf4994066" +dependencies = [ + "futures-core", + "pin-project-lite 0.2.6", + "tokio 1.5.0", +] + [[package]] name = "tokio-util" version = "0.3.1" @@ -2833,6 +3293,20 @@ dependencies = [ "tokio 0.2.25", ] +[[package]] +name = "tokio-util" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.6", + "tokio 1.5.0", +] + [[package]] name = "toml" version = "0.5.8" @@ -2842,6 +3316,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + [[package]] name = "tracing" version = "0.1.26" @@ -2912,6 +3392,12 @@ dependencies = [ "trust-dns-proto", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.13.0" @@ -2963,6 +3449,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "1.7.2" @@ -3051,6 +3543,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -3070,6 +3572,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if 1.0.0", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -3139,6 +3643,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "wepoll-sys" version = "3.0.1" @@ -3164,6 +3678,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" +[[package]] +name = "wildmatch" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f44b95f62d34113cf558c93511ac93027e03e9c29a60dd0fd70e6e025c7270a" + [[package]] name = "winapi" version = "0.2.8" @@ -3216,6 +3736,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -3232,6 +3761,34 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "yup-oauth2" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2573621fa28865489bdf556cdb8703604f4e8498612e3041a212ac20e59c4aeb" +dependencies = [ + "base64 0.13.0", + "chrono", + "futures", + "http", + "hyper", + "hyper-rustls", + "log", + "percent-encoding 2.1.0", + "rustls", + "seahash", + "serde", + "serde_json", + "tokio 1.5.0", + "url 2.2.2", +] + [[package]] name = "zeroize" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 08021637d7..7fa8119421 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,12 @@ authors = [ ] publish = false +# Main server monolith [[bin]] name = "storyteller-web" path = "src/main.rs" +# Jobs for managing downloads and inference [[bin]] name = "tts-download-job" path = "src/tts_download_job.rs" @@ -30,11 +32,13 @@ path = "src/w2l_inference_job.rs" [dependencies] anyhow = "1.0.40" +async-std = { version = "1.6", features = [ "attributes" ] } base64 = "0.13.0" chrono = { version = "0.4.19", features = ["serde"] } derive_more = "0.99.14" easyenv = "0.0.5" futures = "0.3.15" +hex = "0.4.3" hostname = "0.3.1" lazy_static = "1.4.0" log = "0.4.14" @@ -48,6 +52,10 @@ time = "0.2.26" # https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 sqlx = { version = "0.5", features = [ "mysql", "runtime-async-std-native-tls", "offline", "chrono" ] } +# Files +google-drive = "0.1.18" +rust-s3 = "0.27.0-rc4" + # Web actix-cors = "0.5.4" actix-files= "0.5" @@ -60,6 +68,7 @@ jwt = "0.13.0" # Crypto bcrypt = "0.8" hmac = "0.10.0" +md-5 = "0.9.1" sha2 = "0.9.5" # Serialization diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 559e546bd4..2a248a06a3 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -1,7 +1,133 @@ +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + pub mod util; use crate::util::anyhow_result::AnyhowResult; +use std::time::Duration; +use log::info; +use chrono::Utc; +use sqlx::mysql::MySqlPoolOptions; +use sqlx::MySqlPool; + +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; + +#[async_std::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("storyteller-web-unknown".to_string()); + + info!("Hostname: {}", &server_hostname); + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + "mysql://root:root@localhost/storyteller"); + + info!("Connecting to database..."); + + let pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + + loop { + query_and_download(&pool).await; + std::thread::sleep(Duration::from_millis(500)); + } -fn main() -> AnyhowResult<()> { Ok(()) } + +/// table: tts_model_upload_jobs +#[derive(Debug)] +pub struct JobRecord { + pub id: i64, + pub uuid_idempotency_token: String, + pub creator_user_token: String, + pub creator_ip_address: String, + pub creator_set_visibility: String, // TODO + pub title: String, + pub tts_model_type: String, // TODO + pub maybe_subject_token: Option, + pub maybe_actor_subject_token: Option, + pub download_url: Option, + pub download_url_type: String, // TODO + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_and_download(pool: &MySqlPool) { + + let job_records = sqlx::query_as!( + JobRecord, + r#" +SELECT * +FROM tts_model_upload_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT 20 + "#, + ) + .fetch_all(pool) + .await; // TODO: This will return error if it doesn't exist + + info!("Job records..."); + for job_record in job_records { + info!("job record: {:?}", job_record); + } +/* + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_template_upload_jobs +SET + uuid_idempotency_token = ?, + creator_user_token = ?, + creator_ip_address = ?, + creator_set_visibility = ?, + title = ?, + template_type = ?, + maybe_subject_token = ?, + maybe_actor_subject_token = ?, + download_url = ?, + download_url_type = ?, + status = "pending" + "#, + uuid.to_string(), + user_session.user_token.to_string(), + ip_address.to_string(), + creator_set_visibility.to_string(), + title.to_string(), + template_type, + maybe_subject_token, + maybe_actor_subject_token, + download_url, + download_url_type + ) + .execute(&server_state.mysql_pool) + .await; + */ +} \ No newline at end of file