From ec35ad5f6648550b0253e243a4745f6906ad5167 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 00:01:42 -0400 Subject: [PATCH 01/19] init repo --- .gitignore | 5 +++++ README.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..a9a3e92446 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*~ +.idea/ +secrets.toml +secrets.yaml +target/ diff --git a/README.md b/README.md new file mode 100644 index 0000000000..cd7a86141d --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +twitter-firehose +================ + +Ingest events from Twitter and send them to Redis PubSub + From ac78cb3f75734532e9d79161f07b5c177e4f83eb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 01:56:16 -0400 Subject: [PATCH 02/19] Adapting example from egg crate + twitch-firehose (WIP) --- Cargo.lock | 2046 +++++++++++++++++++++++++++++++++++ Cargo.toml | 32 + src/clients/mod.rs | 1 + src/clients/redis_client.rs | 65 ++ src/main.rs | 76 ++ src/secrets.rs | 49 + src/util.rs | 184 ++++ 7 files changed, 2453 insertions(+) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/clients/mod.rs create mode 100644 src/clients/redis_client.rs create mode 100644 src/main.rs create mode 100644 src/secrets.rs create mode 100644 src/util.rs diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..a729662bb7 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2046 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" + +[[package]] +name = "arc-swap" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" + +[[package]] +name = "async-channel" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2114d64672151c0c5eaa5e131ec84a74f06e1e559830dabba01ca30605d66319" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9586ec52317f36de58453159d48351bc244bc24ced3effc1fce22f3d48664af6" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-mutex", + "blocking", + "futures-lite", + "num_cpus", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bbfd5cf2794b1e908ea8457e6c45f8f8f1f6ec5f74617bf4662623f47503c3b" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "once_cell", + "parking", + "polling", + "slab", + "socket2 0.4.0", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-lock" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a8ea61bf9947a1007c5cada31e647dbc77b103c679858150003ba697ea798b" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-mutex" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-native-tls" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9e7a929bd34c68a82d58a4de7f86fffdaf97fb2af850162a7bb19dd7269b33" +dependencies = [ + "async-std", + "native-tls", + "thiserror", + "url", +] + +[[package]] +name = "async-std" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" +dependencies = [ + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "num_cpus", + "once_cell", + "pin-project-lite 0.2.6", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" + +[[package]] +name = "async-trait" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time", + "winapi 0.3.9", +] + +[[package]] +name = "combine" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e" +dependencies = [ + "bytes 0.5.6", + "bytes 1.0.1", + "futures-util", + "memchr", + "pin-project-lite 0.2.6", + "tokio", +] + +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "cpufeatures" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec1028182c380cc45a2e2c5ec841134f2dfd0f8f5f0a5bcd68004f81b5efdf4" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctor" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "0.99.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "easyenv" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516fc6eb3e59cee34859122f05f0c94e28266c76b2ed0efe8d6611ed9c2082c1" +dependencies = [ + "env_logger", + "log", +] + +[[package]] +name = "egg-mode" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f877bc908325f50163ff1670a4733eabf87942511ccfd907fbef4e239c3f8aa" +dependencies = [ + "base64", + "chrono", + "derive_more", + "futures", + "hmac", + "hyper", + "hyper-tls", + "lazy_static", + "mime", + "native-tls", + "percent-encoding", + "rand 0.7.3", + "regex", + "serde", + "serde_json", + "sha-1", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fastrand" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" +dependencies = [ + "instant", +] + +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" + +[[package]] +name = "futures-executor" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" + +[[package]] +name = "futures-lite" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite 0.2.6", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" + +[[package]] +name = "futures-task" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" + +[[package]] +name = "futures-util" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite 0.2.6", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "http" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +dependencies = [ + "bytes 1.0.1", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http", +] + +[[package]] +name = "httparse" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a6f157065790a3ed2f88679250419b5cdd96e714a0d65f7797fd337186e96bb" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project", + "socket2 0.3.19", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper", + "native-tls", + "tokio", + "tokio-tls", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d99f9e3e84b8f67f846ef5b4cbbc3b1c29f6c759fcbce6f01aa0e73d932a24c" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", + "value-bag", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log", + "mio", + "miow 0.3.7", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" + +[[package]] +name = "pin-project-lite" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "polling" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "log", + "wepoll-sys", + "winapi 0.3.9", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "prost" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e6984d2f1a23009bd270b8bb56d0926810a3d483f59c987d77969e9d8e840b2" +dependencies = [ + "bytes 1.0.1", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d3ebd75ac2679c2af3a92246639f9fcc8a442ee420719cc4fe195b98dd5fa3" +dependencies = [ + "bytes 1.0.1", + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "169a15f3008ecb5160cba7d37bcd690a7601b6d30cfb87a117d45e59d52af5d4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b518d7cdd93dab1d1122cf07fa9a60771836c668dde9d9e2a139f957f0d9f1bb" +dependencies = [ + "bytes 1.0.1", + "prost", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "redis" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95357caf2640abc54651b93c98a8df4fe1ccbf44b8e601ccdf43d5c1451f29ac" +dependencies = [ + "arc-swap", + "async-native-tls", + "async-std", + "async-trait", + "bytes 0.5.6", + "combine", + "dtoa", + "futures", + "futures-util", + "itoa", + "native-tls", + "percent-encoding", + "pin-project-lite 0.1.12", + "sha1", + "tokio", + "tokio-tls", + "tokio-util", + "url", +] + +[[package]] +name = "redox_syscall" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + +[[package]] +name = "rustls" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659df5fc3ce22274daac600ffb845300bd2125bcfaec047823075afdab81c00" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "socket2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + +[[package]] +name = "syn" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite 0.1.12", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.12", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +dependencies = [ + "cfg-if 1.0.0", + "log", + "pin-project-lite 0.2.6", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "twitter-firehose" +version = "0.0.0" +dependencies = [ + "anyhow", + "easyenv", + "egg-mode", + "log", + "prost", + "prost-build", + "redis", + "rustls", + "serde", + "serde_derive", + "tokio", + "toml", +] + +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "value-bag" +version = "1.0.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b676010e055c99033117c2343b33a40a30b91fecd6c49055ac9cd2d6c305ab1" +dependencies = [ + "ctor", +] + +[[package]] +name = "vcpkg" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83240549659d187488f91f33c0f8547cbfef0b2088bc470c116d1d260ef623d9" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae70622411ca953215ca6d06d3ebeb1e915f0f6613e3b495122878d7ebec7dae" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b8b767af23de6ac18bf2168b690bed2902743ddf0fb39252e36f9e2bfc63ea" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e734d91443f177bfdb41969de821e15c516931c3c3db3d318fa1b68975d0f6f" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a543ae66aa233d14bb765ed9af4a33e81b8b58d1584cf1b47ff8cd0b9e4489" + +[[package]] +name = "web-sys" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a905d57e488fec8861446d3393670fb50d27a262344013181c2cdf9fff5481be" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "wepoll-sys" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff" +dependencies = [ + "cc", +] + +[[package]] +name = "which" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55551e42cbdf2ce2bedd2203d0cc08dba002c27510f86dab6d0ce304cba3dfe" +dependencies = [ + "either", + "libc", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..6c38d854e4 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "twitter-firehose" +version = "0.0.0" +edition = "2018" +authors = [ + "Brandon Thomas ", + "Brandon Thomas " +] + +[[bin]] +name = "twitter-firehose" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.38" +easyenv = "0.0.4" +log = "0.4.14" +prost = "0.7.0" +rustls = "0.18" +serde = "1.0.123" +serde_derive = "1.0.123" +tokio = { version = "0.2", features = ["full"] } +toml = "0.5.8" + +redis = { version = "0.17.0", features = ["tls", "async-std-tls-comp", "tokio-tls-comp", "connection-manager"] } + +# Twitter +egg-mode = "0.15.0" + +[build-dependencies] +anyhow = "1.0.38" +prost-build = "0.7.0" diff --git a/src/clients/mod.rs b/src/clients/mod.rs new file mode 100644 index 0000000000..66a9cdfc22 --- /dev/null +++ b/src/clients/mod.rs @@ -0,0 +1 @@ +pub mod redis_client; \ No newline at end of file diff --git a/src/clients/redis_client.rs b/src/clients/redis_client.rs new file mode 100644 index 0000000000..28164ecda3 --- /dev/null +++ b/src/clients/redis_client.rs @@ -0,0 +1,65 @@ +use anyhow::anyhow; +use crate::AnyhowResult; +use crate::secrets::RedisSecrets; +use log::{debug, info}; +use redis::AsyncCommands; +use redis::aio::{Connection, ConnectionManager}; +use redis; +use std::thread; +use std::time::Duration; + +pub struct RedisClient { + secrets: RedisSecrets, + + /// This manages our connection. + /// It's meant to be absent when not connected. + connection_manager: Option, + + /// Attempts to retry a Redis command. + max_retry_count: u32, +} + +impl RedisClient { + pub fn new(secrets: &RedisSecrets, retry_count: u32) -> Self { + Self { + secrets: secrets.clone(), + connection_manager: None, + max_retry_count: retry_count, + } + } + + pub async fn connect(&mut self) -> AnyhowResult<()> { + let client = redis::Client::open(self.secrets.connection_url())?; + + let connection_manager = client.get_tokio_connection_manager().await?; + self.connection_manager = Some(connection_manager); + + Ok(()) + } + + pub async fn publish_bytes(&mut self, channel: &str, message: &[u8]) -> AnyhowResult { + for i in 0 .. self.max_retry_count { + match self.try_publish_bytes(channel, message).await { + Ok(r) => return Ok(r), + Err(_) => { + debug!("Redis publish_bytes failed; retrying (attempt {})", i); + continue; + }, + } + } + Err(anyhow!("Retry count elapsed.")) + } + + async fn try_publish_bytes(&mut self, channel: &str, message: &[u8]) -> AnyhowResult { + let connection_manager = match &mut self.connection_manager { + None => { + self.connect().await?; + return Err(anyhow!("Not connected")); + }, + Some(cm) => cm, + }; + + let result = connection_manager.publish(channel, message).await?; + Ok(result) + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000000..ad107a4703 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,76 @@ +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + +#[macro_use] +extern crate serde_derive; + +mod clients; +mod secrets; +//mod util; + +//use futures::TryStreamExt; +use egg_mode::stream::StreamMessage; +use crate::secrets::Secrets; +use crate::clients::redis_client::RedisClient; + +pub type AnyhowResult = anyhow::Result; + +const ENV_PUBLISH_TOPIC : &'static str = "PUBLISH_TOPIC"; +const ENV_PUBLISH_TOPIC_DEFAULT : &'static str = "firehose"; + +const ENV_REDIS_MAX_RETRY_COUNT : &'static str = "REDIS_MAX_RETRY_COUNT"; +const ENV_REDIS_MAX_RETRY_COUNT_DEFAULT : u32 = 3; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + + easyenv::init_env_logger(None); + + let redis_publish_topic = easyenv::get_env_string_or_default( + ENV_PUBLISH_TOPIC, ENV_PUBLISH_TOPIC_DEFAULT); + + let redis_max_retry_count = easyenv::get_env_num( + ENV_REDIS_MAX_RETRY_COUNT, + ENV_REDIS_MAX_RETRY_COUNT_DEFAULT)?; + + let secrets = Secrets::from_file("secrets.toml")?; + + let mut redis_client = RedisClient::new( + &secrets.redis, + redis_max_retry_count + ); + + redis_client.connect().await?; + + // We can capture: Mentions, Retweets, Replies. + // We cannot capture: Follows. + + /*let config = common::Config::load().await; + println!("Streaming tweets containing popular programming languages (and also Rust)"); + println!("Ctrl-C to quit\n"); + + let stream = egg_mode::stream::filter() + .follow(&[1297106371238932481]) // vocodes + //.track(&["rustlang", "python", "java", "javascript"]) + .language(&["en"]) + .start(&config.token) + .try_for_each(|m| { + if let StreamMessage::Tweet(tweet) = m { + common::print_tweet(&tweet); + println!("──────────────────────────────────────"); + } else { + println!("{:?}", m); + } + futures::future::ok(()) + }); + if let Err(e) = stream.await { + println!("Stream error: {}", e); + println!("Disconnected") + } +*/ + + Ok(()) +} \ No newline at end of file diff --git a/src/secrets.rs b/src/secrets.rs new file mode 100644 index 0000000000..1270514896 --- /dev/null +++ b/src/secrets.rs @@ -0,0 +1,49 @@ +use crate::AnyhowResult; +use std::io::Read; + +#[derive(Deserialize)] +pub struct Secrets { + pub redis: RedisSecrets, + pub twitter: TwitterSecrets, +} + +#[derive(Deserialize, Clone)] +pub struct RedisSecrets { + pub username: String, + pub password: String, + pub host: String, + pub port: u32, +} + +#[derive(Deserialize, Clone)] +pub struct TwitterSecrets { + pub api_key: String, + pub api_secret_key: String, +} + +impl Secrets { + pub fn from_file(filename: &str) -> AnyhowResult { + let mut file = std::fs::File::open(filename)?; + let mut buffer = String::new(); + file.read_to_string(&mut buffer)?; + let secrets = toml::from_str(&buffer)?; + Ok(secrets) + } +} + +impl RedisSecrets { + pub fn connection_url(&self) -> String { + format!("rediss://{}:{}@{}:{}", self.username, self.password, self.host, self.port) + } +} + +/*impl TwitterSecrets { + pub fn get_user_config(&self) -> AnyhowResult { + let config = UserConfig::builder() + .name(self.username.to_string()) + .token(self.oauth_access_token.to_string()) + .enable_all_capabilities() + .build()?; + Ok(config) + } +}*/ diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000000..586312e56c --- /dev/null +++ b/src/util.rs @@ -0,0 +1,184 @@ + +pub struct Config { + pub token: egg_mode::Token, + pub user_id: u64, + pub screen_name: String, +} + +impl Config { + pub async fn load() -> Self { + let a1 = Config::load_inner().await; + if let Some(conf) = a1 { + return conf; + } + + Config::load_inner().await.unwrap() + } + + /// This needs to be a separate function so we can retry after creating the + /// twitter_settings file. Idealy we would recurse, but that requires boxing + /// the output which doesn't seem worthwhile + async fn load_inner() -> Option { + //IMPORTANT: make an app for yourself at apps.twitter.com and get your + //key/secret into these files; these examples won't work without them + let consumer_key = include_str!("consumer_key").trim(); + let consumer_secret = include_str!("consumer_secret").trim(); + + let con_token = egg_mode::KeyPair::new(consumer_key, consumer_secret); + + let mut config = String::new(); + let user_id: u64; + let username: String; + let token: egg_mode::Token; + + //look at all this unwrapping! who told you it was my birthday? + if let Ok(mut f) = std::fs::File::open("twitter_settings") { + f.read_to_string(&mut config).unwrap(); + + let mut iter = config.split('\n'); + + username = iter.next().unwrap().to_string(); + user_id = u64::from_str_radix(&iter.next().unwrap(), 10).unwrap(); + let access_token = egg_mode::KeyPair::new( + iter.next().unwrap().to_string(), + iter.next().unwrap().to_string(), + ); + token = egg_mode::Token::Access { + consumer: con_token, + access: access_token, + }; + + if let Err(err) = egg_mode::auth::verify_tokens(&token).await { + println!("We've hit an error using your old tokens: {:?}", err); + println!("We'll have to reauthenticate before continuing."); + std::fs::remove_file("twitter_settings").unwrap(); + } else { + println!("Welcome back, {}!\n", username); + } + } else { + let request_token = egg_mode::auth::request_token(&con_token, "oob").await.unwrap(); + + println!("Go to the following URL, sign in, and give me the PIN that comes back:"); + println!("{}", egg_mode::auth::authorize_url(&request_token)); + + let mut pin = String::new(); + std::io::stdin().read_line(&mut pin).unwrap(); + println!(""); + + let tok_result = egg_mode::auth::access_token(con_token, &request_token, pin) + .await + .unwrap(); + + token = tok_result.0; + user_id = tok_result.1; + username = tok_result.2; + + match token { + egg_mode::Token::Access { + access: ref access_token, + .. + } => { + config.push_str(&username); + config.push('\n'); + config.push_str(&format!("{}", user_id)); + config.push('\n'); + config.push_str(&access_token.key); + config.push('\n'); + config.push_str(&access_token.secret); + } + _ => unreachable!(), + } + + let mut f = std::fs::File::create("twitter_settings").unwrap(); + f.write_all(config.as_bytes()).unwrap(); + + println!("Welcome, {}, let's get this show on the road!", username); + } + + //TODO: Is there a better way to query whether a file exists? + if std::fs::metadata("twitter_settings").is_ok() { + Some(Config { + token: token, + user_id: user_id, + screen_name: username, + }) + } else { + None + } + } +} + + +pub fn print_tweet(tweet: &egg_mode::tweet::Tweet) { + if let Some(ref user) = tweet.user { + println!( + "{} (@{}) posted at {}", + Paint::blue(&user.name), + Paint::bold(Paint::blue(&user.screen_name)), + tweet.created_at.with_timezone(&chrono::Local) + ); + } + + if let Some(ref screen_name) = tweet.in_reply_to_screen_name { + println!("➜ in reply to @{}", Paint::blue(screen_name)); + } + + if let Some(ref status) = tweet.retweeted_status { + println!("{}", Paint::red("Retweet ➜")); + print_tweet(status); + return; + } else { + println!("{}", Paint::green(&tweet.text)); + } + + if let Some(source) = &tweet.source { + println!("➜ via {} ({})", source.name, source.url); + } + + if let Some(ref place) = tweet.place { + println!("➜ from: {}", place.full_name); + } + + if let Some(ref status) = tweet.quoted_status { + println!("{}", Paint::red("➜ Quoting the following status:")); + print_tweet(status); + } + + if !tweet.entities.hashtags.is_empty() { + println!("➜ Hashtags contained in the tweet:"); + for tag in &tweet.entities.hashtags { + println!(" {}", tag.text); + } + } + + if !tweet.entities.symbols.is_empty() { + println!("➜ Symbols contained in the tweet:"); + for tag in &tweet.entities.symbols { + println!(" {}", tag.text); + } + } + + if !tweet.entities.urls.is_empty() { + println!("➜ URLs contained in the tweet:"); + for url in &tweet.entities.urls { + if let Some(expanded_url) = &url.expanded_url { + println!(" {}", expanded_url); + } + } + } + + if !tweet.entities.user_mentions.is_empty() { + println!("➜ Users mentioned in the tweet:"); + for user in &tweet.entities.user_mentions { + println!(" {}", Paint::bold(Paint::blue(&user.screen_name))); + } + } + + if let Some(ref media) = tweet.extended_entities { + println!("➜ Media attached to the tweet:"); + for info in &media.media { + println!(" A {:?}", info.media_type); + } + } +} + From cc20bee7277c195a03f635ac50a69b9a041e82a2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 14:21:14 -0400 Subject: [PATCH 03/19] WIP, work on Twitter secrets --- src/main.rs | 17 +++++----- src/secrets.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index ad107a4703..6a6fb136b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,9 +24,12 @@ const ENV_PUBLISH_TOPIC_DEFAULT : &'static str = "firehose"; const ENV_REDIS_MAX_RETRY_COUNT : &'static str = "REDIS_MAX_RETRY_COUNT"; const ENV_REDIS_MAX_RETRY_COUNT_DEFAULT : u32 = 3; +/// Monitor Twitter for interesting events against our account. +/// With this implementation, we can capture: Mentions, Retweets, Replies. +/// We cannot capture: Follows. #[tokio::main] -async fn main() -> anyhow::Result<()> { - +async fn main() -> anyhow::Result<()> +{ easyenv::init_env_logger(None); let redis_publish_topic = easyenv::get_env_string_or_default( @@ -45,12 +48,10 @@ async fn main() -> anyhow::Result<()> { redis_client.connect().await?; - // We can capture: Mentions, Retweets, Replies. - // We cannot capture: Follows. - /*let config = common::Config::load().await; - println!("Streaming tweets containing popular programming languages (and also Rust)"); - println!("Ctrl-C to quit\n"); + //let config = common::Config::load().await; + //println!("Streaming tweets containing popular programming languages (and also Rust)"); + //println!("Ctrl-C to quit\n"); let stream = egg_mode::stream::filter() .follow(&[1297106371238932481]) // vocodes @@ -66,11 +67,11 @@ async fn main() -> anyhow::Result<()> { } futures::future::ok(()) }); + if let Err(e) = stream.await { println!("Stream error: {}", e); println!("Disconnected") } -*/ Ok(()) } \ No newline at end of file diff --git a/src/secrets.rs b/src/secrets.rs index 1270514896..34def71c39 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -19,6 +19,10 @@ pub struct RedisSecrets { pub struct TwitterSecrets { pub api_key: String, pub api_secret_key: String, + + // These have to be obtained over an OAuth-like flow + pub access_key: Option, + pub access_secret: Option, } impl Secrets { @@ -37,13 +41,76 @@ impl RedisSecrets { } } -/*impl TwitterSecrets { - pub fn get_user_config(&self) -> AnyhowResult { - let config = UserConfig::builder() - .name(self.username.to_string()) - .token(self.oauth_access_token.to_string()) - .enable_all_capabilities() - .build()?; - Ok(config) +impl TwitterSecrets { + + pub fn make_consumer_keypair(&self) -> egg_mode::KeyPair { + egg_mode::KeyPair::new(self.api_key.to_string(), self.api_secret_key.to_string()) } -}*/ + + pub fn make_access_keypair(&self) -> AnyhowResult { + let access_key = self.access_key + .map(|s| s.clone()) + .ok_or(anyhow!("No access key!"))?; + + let access_secret = self.access_secret + .map(|s| s.clone()) + .ok_or(anyhow!("No access secret!"))?; + + Ok(egg_mode::KeyPair::new(access_key, access_secret)) + } + + pub fn get_access_token(&self) -> AnyhowResult { + let consumer_keypair = self.make_consumer_keypair(); + + let access_keypair = self.make_access_keypair()?; + + let access_token = egg_mode::Token::Access { + consumer: consumer_keypair, + access: access_keypair, + }; + + egg_mode::auth::verify_tokens(&access_token).await?; + + Ok(access_token) + } + + /// Call this to get an access token. + pub fn request_access_token(&self) -> AnyhowResult<()> { + + let connection_token = egg_mode::KeyPair::new(consumer_key, consumer_secret); + + let request_token = egg_mode::auth::request_token(&connection_token, "oob") + .await?; + + warn!("Go to the following URL, sign in, and paste the PIN that comes back:"); + warn!("{}", egg_mode::auth::authorize_url(&request_token)); + + let mut pin = String::new(); + std::io::stdin().read_line(&mut pin)?; + + let token_result = egg_mode::auth::access_token(connection_token, &request_token, pin) + .await?; + + token = token_result.0; + user_id = token_result.1; + username = token_result.2; + + /*match token { + egg_mode::Token::Access { + access: ref access_token, + .. + } => { + config.push_str(&username); + config.push('\n'); + config.push_str(&format!("{}", user_id)); + config.push('\n'); + config.push_str(&access_token.key); + config.push('\n'); + config.push_str(&access_token.secret); + } + _ => unreachable!(), + }*/ + + Ok(()) + } +} From 3d920f9791ec79507fceb63a878484b5254572ac Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 14:38:53 -0400 Subject: [PATCH 04/19] get_ccess_token works! --- Cargo.toml | 4 ++ src/main_get_access_token.rs | 44 ++++++++++++++++++++ src/secrets.rs | 78 ++++++++++++++++++------------------ 3 files changed, 88 insertions(+), 38 deletions(-) create mode 100644 src/main_get_access_token.rs diff --git a/Cargo.toml b/Cargo.toml index 6c38d854e4..2dc7b8c9b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,10 @@ authors = [ name = "twitter-firehose" path = "src/main.rs" +[[bin]] +name = "twitter-get-acccess-token" +path = "src/main_get_access_token.rs" + [dependencies] anyhow = "1.0.38" easyenv = "0.0.4" diff --git a/src/main_get_access_token.rs b/src/main_get_access_token.rs new file mode 100644 index 0000000000..c6012f1aed --- /dev/null +++ b/src/main_get_access_token.rs @@ -0,0 +1,44 @@ +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + +#[macro_use] +extern crate serde_derive; + +mod secrets; + +//use futures::TryStreamExt; +use egg_mode::stream::StreamMessage; +use crate::secrets::Secrets; + +pub type AnyhowResult = anyhow::Result; + +const ENV_PUBLISH_TOPIC : &'static str = "PUBLISH_TOPIC"; +const ENV_PUBLISH_TOPIC_DEFAULT : &'static str = "firehose"; + +const ENV_REDIS_MAX_RETRY_COUNT : &'static str = "REDIS_MAX_RETRY_COUNT"; +const ENV_REDIS_MAX_RETRY_COUNT_DEFAULT : u32 = 3; + +/// Monitor Twitter for interesting events against our account. +/// With this implementation, we can capture: Mentions, Retweets, Replies. +/// We cannot capture: Follows. +#[tokio::main] +async fn main() -> anyhow::Result<()> +{ + easyenv::init_env_logger(None); + + let redis_publish_topic = easyenv::get_env_string_or_default( + ENV_PUBLISH_TOPIC, ENV_PUBLISH_TOPIC_DEFAULT); + + let redis_max_retry_count = easyenv::get_env_num( + ENV_REDIS_MAX_RETRY_COUNT, + ENV_REDIS_MAX_RETRY_COUNT_DEFAULT)?; + + let secrets = Secrets::from_file("secrets.toml")?; + + secrets.twitter.request_access_token().await?; + + Ok(()) +} diff --git a/src/secrets.rs b/src/secrets.rs index 34def71c39..f9c797c7da 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -1,5 +1,7 @@ use crate::AnyhowResult; use std::io::Read; +use log::{info, warn, debug}; +use anyhow::anyhow; #[derive(Deserialize)] pub struct Secrets { @@ -42,26 +44,9 @@ impl RedisSecrets { } impl TwitterSecrets { - - pub fn make_consumer_keypair(&self) -> egg_mode::KeyPair { - egg_mode::KeyPair::new(self.api_key.to_string(), self.api_secret_key.to_string()) - } - - pub fn make_access_keypair(&self) -> AnyhowResult { - let access_key = self.access_key - .map(|s| s.clone()) - .ok_or(anyhow!("No access key!"))?; - - let access_secret = self.access_secret - .map(|s| s.clone()) - .ok_or(anyhow!("No access secret!"))?; - - Ok(egg_mode::KeyPair::new(access_key, access_secret)) - } - - pub fn get_access_token(&self) -> AnyhowResult { + /// This assumes we have a valid pair of access (key, secret). + pub async fn get_access_token(&self) -> AnyhowResult { let consumer_keypair = self.make_consumer_keypair(); - let access_keypair = self.make_access_keypair()?; let access_token = egg_mode::Token::Access { @@ -75,11 +60,10 @@ impl TwitterSecrets { } /// Call this to get an access token. - pub fn request_access_token(&self) -> AnyhowResult<()> { - - let connection_token = egg_mode::KeyPair::new(consumer_key, consumer_secret); - - let request_token = egg_mode::auth::request_token(&connection_token, "oob") + /// This is an interactive flow that requests input from stdin. + pub async fn request_access_token(&self) -> AnyhowResult<()> { + let consumer_keypair = self.make_consumer_keypair(); + let request_token = egg_mode::auth::request_token(&consumer_keypair, "oob") .await?; warn!("Go to the following URL, sign in, and paste the PIN that comes back:"); @@ -88,29 +72,47 @@ impl TwitterSecrets { let mut pin = String::new(); std::io::stdin().read_line(&mut pin)?; - let token_result = egg_mode::auth::access_token(connection_token, &request_token, pin) + let token_result = egg_mode::auth::access_token(consumer_keypair, &request_token, pin) .await?; - token = token_result.0; - user_id = token_result.1; - username = token_result.2; + let access_token = token_result.0; + let user_id = token_result.1; + let username = token_result.2; - /*match token { + warn!("Ordinarily these should not be logged, but there is no log aggregator installed."); + warn!("In any event, it is best to do this locally before creating k8s secrets."); + warn!("user_id = {:?}", user_id); + warn!("username = {:?}", username); + + match access_token { egg_mode::Token::Access { - access: ref access_token, + access: ref access, .. } => { - config.push_str(&username); - config.push('\n'); - config.push_str(&format!("{}", user_id)); - config.push('\n'); - config.push_str(&access_token.key); - config.push('\n'); - config.push_str(&access_token.secret); + warn!("access_token_key = {:?}", access.key); + warn!("access_token_secret = {:?}", access.secret); } _ => unreachable!(), - }*/ + } Ok(()) } + + pub fn make_consumer_keypair(&self) -> egg_mode::KeyPair { + egg_mode::KeyPair::new(self.api_key.to_string(), self.api_secret_key.to_string()) + } + + pub fn make_access_keypair(&self) -> AnyhowResult { + let access_key = self.access_key + .as_ref() + .ok_or(anyhow!("No access key!"))? + .clone(); + + let access_secret = self.access_secret + .as_ref() + .ok_or(anyhow!("No access secret!"))? + .clone(); + + Ok(egg_mode::KeyPair::new(access_key, access_secret)) + } } From 59c71a82d4f1afea2d20380017bb1ba9fb8667ff Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 15:00:39 -0400 Subject: [PATCH 05/19] This works! --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 39 +++++++++++++++++++-------------------- src/secrets.rs | 6 +++--- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a729662bb7..299706b42b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1773,6 +1773,7 @@ dependencies = [ "anyhow", "easyenv", "egg-mode", + "futures", "log", "prost", "prost-build", diff --git a/Cargo.toml b/Cargo.toml index 2dc7b8c9b8..6d980e7e7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ path = "src/main_get_access_token.rs" [dependencies] anyhow = "1.0.38" easyenv = "0.0.4" +futures = "0.3.14" log = "0.4.14" prost = "0.7.0" rustls = "0.18" diff --git a/src/main.rs b/src/main.rs index 6a6fb136b4..466176afb1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,10 +11,11 @@ mod clients; mod secrets; //mod util; -//use futures::TryStreamExt; +use futures::TryStreamExt; use egg_mode::stream::StreamMessage; use crate::secrets::Secrets; use crate::clients::redis_client::RedisClient; +use log::{info, warn, debug}; pub type AnyhowResult = anyhow::Result; @@ -48,29 +49,27 @@ async fn main() -> anyhow::Result<()> redis_client.connect().await?; + info!("Verifying Twitter access token..."); + let twitter_access_token = secrets.twitter.verify_access_token().await?; - //let config = common::Config::load().await; - //println!("Streaming tweets containing popular programming languages (and also Rust)"); - //println!("Ctrl-C to quit\n"); - + info!("Streaming..."); + let stream = egg_mode::stream::filter() - .follow(&[1297106371238932481]) // vocodes - //.track(&["rustlang", "python", "java", "javascript"]) - .language(&["en"]) - .start(&config.token) - .try_for_each(|m| { - if let StreamMessage::Tweet(tweet) = m { - common::print_tweet(&tweet); - println!("──────────────────────────────────────"); - } else { - println!("{:?}", m); - } - futures::future::ok(()) - }); + .follow(&[1297106371238932481]) // vocodes + .track(&["vocodes"]) + //.language(&["en"]) + .start(&twitter_access_token) + .try_for_each(|m| { + if let StreamMessage::Tweet(tweet) = m { + info!("Tweet: {:?}", &tweet); + } else { + info!("Other: {:?}", &m); + } + futures::future::ok(()) + }); if let Err(e) = stream.await { - println!("Stream error: {}", e); - println!("Disconnected") + warn!("Disconnected. Stream error: {:?}", e); } Ok(()) diff --git a/src/secrets.rs b/src/secrets.rs index f9c797c7da..3170a031cd 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -45,7 +45,7 @@ impl RedisSecrets { impl TwitterSecrets { /// This assumes we have a valid pair of access (key, secret). - pub async fn get_access_token(&self) -> AnyhowResult { + pub async fn verify_access_token(&self) -> AnyhowResult { let consumer_keypair = self.make_consumer_keypair(); let access_keypair = self.make_access_keypair()?; @@ -89,8 +89,8 @@ impl TwitterSecrets { access: ref access, .. } => { - warn!("access_token_key = {:?}", access.key); - warn!("access_token_secret = {:?}", access.secret); + warn!("access_key = {:?}", access.key); + warn!("access_secret = {:?}", access.secret); } _ => unreachable!(), } From 14d99b44b79582aed1455cfcbc57a6ccc76e2933 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 15:12:06 -0400 Subject: [PATCH 06/19] mention example --- src/main.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 466176afb1..7ffd615187 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,7 +53,39 @@ async fn main() -> anyhow::Result<()> let twitter_access_token = secrets.twitter.verify_access_token().await?; info!("Streaming..."); - + + /* + + Here's an @vocodes mention: + =========================== + + Tweet: Tweet { coordinates: None, created_at: 2021-05-09T19:00:01Z, current_user_retweet: None, + display_text_range: None, entities: TweetEntities { hashtags: [], symbols: [], urls: [], + + user_mentions: [MentionEntity { id: 1297106371238932481, range: (17, 25), name: "vocodes", screen_name: "vocodes" }], + + media: None }, extended_entities: None, favorite_count: 0, favorited: Some(false), filter_level: Some(Low), + id: 1391468014755880962, in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, + lang: Some("en"), place: None, possibly_sensitive: None, quoted_status_id: None, quoted_status: None, + retweet_count: 0, retweeted: Some(false), retweeted_status: None, + source: Some(TweetSource { name: "Twitter Web App", url: "https://mobile.twitter.com" }), + text: "Okay, so I think @vocodes mentions should work", truncated: false, + user: Some(TwitterUser { contributors_enabled: false, created_at: 2020-09-27T02:04:35Z, default_profile: true, + default_profile_image: false, description: Some("This is a bot account for @vocodes / + https://t.co/EDUojadeKa, created by @echelon. You can make it quote you by mentioning it. Please don\'t use this for evil. DEEP FAKE."), + entities: UserEntities { description: UserEntityDetail { urls: [] }, url: None }, favourites_count: 2, follow_request_sent: None, + followers_count: 30, friends_count: 1, geo_enabled: true, id: 1310037441441599488, is_translator: false, + lang: None, listed_count: 1, location: Some("Atlanta, GA"), name: "vocodes bot", profile_background_color: "F5F8FA", + profile_background_image_url: Some(""), profile_background_image_url_https: Some(""), profile_background_tile: Some(false), + profile_banner_url: None, profile_image_url: "http://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", + profile_image_url_https: "https://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", profile_link_color: "1DA1F2", + profile_sidebar_border_color: "C0DEED", profile_sidebar_fill_color: "DDEEF6", profile_text_color: "333333", + profile_use_background_image: true, protected: false, screen_name: "VocodesBot", show_all_inline_media: None, status: None, + statuses_count: 64, time_zone: None, url: Some("https://vo.codes"), utc_offset: None, verified: false, + withheld_in_countries: Some([]), withheld_scope: None }), withheld_copyright: false, withheld_in_countries: None, withheld_scope: None } + + */ + let stream = egg_mode::stream::filter() .follow(&[1297106371238932481]) // vocodes .track(&["vocodes"]) From 2787fa7d67f7ca05a15bbbfab00b34de0ee41a8b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 9 May 2021 22:59:36 -0400 Subject: [PATCH 07/19] more data --- src/main.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 7ffd615187..f477d7f75c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -56,8 +56,8 @@ async fn main() -> anyhow::Result<()> /* - Here's an @vocodes mention: - =========================== + Here's an @vocodes mention by @vocodesbot: + ========================================== Tweet: Tweet { coordinates: None, created_at: 2021-05-09T19:00:01Z, current_user_retweet: None, display_text_range: None, entities: TweetEntities { hashtags: [], symbols: [], urls: [], @@ -84,6 +84,66 @@ async fn main() -> anyhow::Result<()> statuses_count: 64, time_zone: None, url: Some("https://vo.codes"), utc_offset: None, verified: false, withheld_in_countries: Some([]), withheld_scope: None }), withheld_copyright: false, withheld_in_countries: None, withheld_scope: None } + + Here's an @vocodes retweet (not quote tweet) by @vocodesbot + ============================================================ + + Tweet: Tweet { coordinates: None, created_at: 2021-05-09T19:12:44Z, current_user_retweet: None, display_text_range: None, + entities: TweetEntities { hashtags: [], symbols: [], urls: [], + + user_mentions: [MentionEntity { id: 1297106371238932481, range: (3, 11), name: "vocodes", screen_name: "vocodes" }], + + media: None }, extended_entities: None, favorite_count: 0, favorited: Some(false), filter_level: Some(Low), id: 1391471213713805315, + in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, lang: Some("en"), place: None, + possibly_sensitive: None, quoted_status_id: None, quoted_status: None, retweet_count: 0, retweeted: Some(false), + + retweeted_status: Some(Tweet { coordinates: None, created_at: 2021-04-25T02:49:40Z, current_user_retweet: None, display_text_range: Some((0, 140)), + entities: TweetEntities { hashtags: [], symbols: [], urls: [UrlEntity { display_url: "twitter.com/i/web/status/1…", + + expanded_url: Some("https://twitter.com/i/web/status/1386150388873715712"), range: (117, 140), + url: "https://t.co/x5JPIBKr3x" }], user_mentions: [], media: None }, extended_entities: None, + favorite_count: 17, favorited: Some(false), filter_level: Some(Low), id: 1386150388873715712, + in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, lang: Some("en"), + place: None, possibly_sensitive: Some(false), quoted_status_id: None, quoted_status: None, retweet_count: 11, + retweeted: Some(false), retweeted_status: None, source: Some(TweetSource { name: "Twitter Web App", + url: "https://mobile.twitter.com" }), + text: "You\'ll also get Joker (and JFK) in tomorrow\'s stream. I haven\'t scheduled it yet, but I\'m + thinking 2 or 3 PM EDT. I\'ll post in a bit. https://t.co/T97CR9BXcf", + truncated: true, user: Some(TwitterUser { contributors_enabled: false, created_at: 2020-08-22T09:41:05Z, + default_profile: true, default_profile_image: false, description: Some("It looks like a toy now, but + I want to democratize Hollywood with tech. \ntext to speech: https://t.co/htL5ArOIPW\ninteractive + twitch stream: https://t.co/qMvlOF5Zji\nMade by @echelon"), entities: UserEntities { description: + UserEntityDetail { urls: [] }, url: None }, favourites_count: 602, follow_request_sent: None, + followers_count: 2264, friends_count: 129, geo_enabled: false, id: 1297106371238932481, is_translator: false, + lang: None, listed_count: 7, location: Some("Atlanta, GA"), name: "vocodes", profile_background_color: "F5F8FA", + profile_background_image_url: Some(""), profile_background_image_url_https: Some(""), + profile_background_tile: Some(false), profile_banner_url: Some("https://pbs.twimg.com/profile_banners/1297106371238932481/1599623560"), + profile_image_url: "http://pbs.twimg.com/profile_images/1305039188023222273/NmVznkUO_normal.png", + profile_image_url_https: "https://pbs.twimg.com/profile_images/1305039188023222273/NmVznkUO_normal.png", + profile_link_color: "1DA1F2", profile_sidebar_border_color: "C0DEED", profile_sidebar_fill_color: "DDEEF6", + profile_text_color: "333333", profile_use_background_image: true, protected: false, screen_name: "vocodes", + show_all_inline_media: None, status: None, statuses_count: 303, time_zone: None, url: Some("https://vo.codes"), + utc_offset: None, verified: false, withheld_in_countries: Some([]), withheld_scope: None }), + withheld_copyright: false, withheld_in_countries: None, withheld_scope: None }), + source: Some(TweetSource { name: "Twitter Web App", url: "https://mobile.twitter.com" }), + text: "RT @vocodes: You\'ll also get Joker (and JFK) in tomorrow\'s stream. I haven\'t scheduled it yet, but + I\'m thinking 2 or 3 PM EDT. I\'ll post in…", truncated: false, user: Some(TwitterUser { contributors_enabled: false, + created_at: 2020-09-27T02:04:35Z, default_profile: true, default_profile_image: false, description: Some("This is a bot + account for @vocodes / https://t.co/EDUojadeKa, created by @echelon. You can make it quote you by mentioning it. + Please don\'t use this for evil. DEEP FAKE."), entities: UserEntities { description: + UserEntityDetail { urls: [] }, url: None }, favourites_count: 2, follow_request_sent: None, followers_count: 30, + friends_count: 1, geo_enabled: true, id: 1310037441441599488, is_translator: false, lang: None, listed_count: 1, + location: Some("Atlanta, GA"), name: "vocodes bot", profile_background_color: "F5F8FA", profile_background_image_url: Some(""), + profile_background_image_url_https: Some(""), profile_background_tile: Some(false), profile_banner_url: None, + profile_image_url: "http://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", + profile_image_url_https: "https://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", + profile_link_color: "1DA1F2", profile_sidebar_border_color: "C0DEED", profile_sidebar_fill_color: "DDEEF6", + profile_text_color: "333333", profile_use_background_image: true, protected: false, screen_name: "VocodesBot", + show_all_inline_media: None, status: None, statuses_count: 65, time_zone: None, url: Some("https://vo.codes"), utc_offset: None, + verified: false, withheld_in_countries: Some([]), withheld_scope: None }), withheld_copyright: false, withheld_in_countries: None, + withheld_scope: None } + + */ let stream = egg_mode::stream::filter() From 7b9e122eb032c533779b0f2cff76d7426e28cd2d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 02:16:57 -0400 Subject: [PATCH 08/19] self-managing twitter client --- src/clients/mod.rs | 3 ++- src/clients/twitter_client.rs | 46 +++++++++++++++++++++++++++++++++++ src/main.rs | 7 ++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 src/clients/twitter_client.rs diff --git a/src/clients/mod.rs b/src/clients/mod.rs index 66a9cdfc22..9ddc2bd5bf 100644 --- a/src/clients/mod.rs +++ b/src/clients/mod.rs @@ -1 +1,2 @@ -pub mod redis_client; \ No newline at end of file +pub mod redis_client; +pub mod twitter_client; diff --git a/src/clients/twitter_client.rs b/src/clients/twitter_client.rs new file mode 100644 index 0000000000..70677155a6 --- /dev/null +++ b/src/clients/twitter_client.rs @@ -0,0 +1,46 @@ +use crate::AnyhowResult; +use log::{info, debug, warn}; +use futures::TryStreamExt; +use egg_mode::stream::StreamMessage; +use std::time::Duration; +use std::thread; + +pub struct TwitterClient { + access_token: egg_mode::Token, +} + +impl TwitterClient { + + pub fn new(access_token: egg_mode::Token) -> Self { + Self { + access_token, + } + } + + pub async fn main_loop(&self) { + loop { + if let Err(e) = self.run_stream().await { + warn!("Disconnected from stream; error = {:?}", e); + } + info!("Reconnecting after timeout..."); + thread::sleep(Duration::from_secs(30)); + } + } + + pub async fn run_stream(&self) -> AnyhowResult<()> { + let stream = egg_mode::stream::filter() + .follow(&[1297106371238932481]) // vocodes + .track(&["vocodes"]) + .start(&self.access_token) + .try_for_each(|m| { + if let StreamMessage::Tweet(tweet) = m { + info!("Tweet: {:?}", &tweet); + } else { + info!("Other (non-Tweet) message: {:?}", &m); + } + futures::future::ok(()) + }); + + Ok(stream.await?) + } +} diff --git a/src/main.rs b/src/main.rs index f477d7f75c..9c6d3e93ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ use egg_mode::stream::StreamMessage; use crate::secrets::Secrets; use crate::clients::redis_client::RedisClient; use log::{info, warn, debug}; +use crate::clients::twitter_client::TwitterClient; pub type AnyhowResult = anyhow::Result; @@ -146,6 +147,7 @@ async fn main() -> anyhow::Result<()> */ + /* let stream = egg_mode::stream::filter() .follow(&[1297106371238932481]) // vocodes .track(&["vocodes"]) @@ -164,5 +166,10 @@ async fn main() -> anyhow::Result<()> warn!("Disconnected. Stream error: {:?}", e); } + Ok(())*/ + + let twitter_client = TwitterClient::new(twitter_access_token); + + twitter_client.main_loop().await; Ok(()) } \ No newline at end of file From 5c8a1730f266c285c13ed6c4441a639a12677eb9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 02:51:46 -0400 Subject: [PATCH 09/19] these are the fields we'll use... --- src/clients/twitter_client.rs | 82 +++++++++++++++++++++++++++++++++-- src/main.rs | 4 +- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/clients/twitter_client.rs b/src/clients/twitter_client.rs index 70677155a6..6645e20b9b 100644 --- a/src/clients/twitter_client.rs +++ b/src/clients/twitter_client.rs @@ -4,6 +4,9 @@ use futures::TryStreamExt; use egg_mode::stream::StreamMessage; use std::time::Duration; use std::thread; +use egg_mode::tweet::Tweet; + +const VOCODES_USER_ID : u64 = 1297106371238932481; pub struct TwitterClient { access_token: egg_mode::Token, @@ -19,22 +22,23 @@ impl TwitterClient { pub async fn main_loop(&self) { loop { + info!("Connecting to twitter stream..."); if let Err(e) = self.run_stream().await { warn!("Disconnected from stream; error = {:?}", e); } - info!("Reconnecting after timeout..."); + info!("Disconnected; reconnecting after timeout..."); thread::sleep(Duration::from_secs(30)); } } pub async fn run_stream(&self) -> AnyhowResult<()> { let stream = egg_mode::stream::filter() - .follow(&[1297106371238932481]) // vocodes + .follow(&[VOCODES_USER_ID]) // vocodes .track(&["vocodes"]) .start(&self.access_token) .try_for_each(|m| { if let StreamMessage::Tweet(tweet) = m { - info!("Tweet: {:?}", &tweet); + self.handle_tweet(tweet); } else { info!("Other (non-Tweet) message: {:?}", &m); } @@ -43,4 +47,76 @@ impl TwitterClient { Ok(stream.await?) } + + fn handle_tweet(&self, tweet: Tweet) -> AnyhowResult<()> { + info!("Tweet: {:?}", &tweet); + + let mut user_id : Option = None; + let mut username : Option = None; + let mut display_name : Option = None; + let mut profile_image_url : Option = None; + + if let Some(user) = tweet.user { + if user.id == VOCODES_USER_ID { + warn!("Tweet is by @vocodes account; skipping"); + return Ok(()); + } + + user_id = Some(user.id); + username = Some(user.screen_name.clone()); + display_name = Some(user.name.clone()); + profile_image_url = Some(user.profile_image_url.clone()); + } + + let mut has_mention = false; + + for mention in tweet.entities.user_mentions.iter() { + if mention.id == VOCODES_USER_ID { + has_mention = true; + } + } + + let mut is_retweet = false; + let mut retweeted_text : Option = None; + + if let Some(retweet) = tweet.retweeted_status { + if let Some(original_author) = retweet.user { + if original_author.id == VOCODES_USER_ID { + is_retweet = true; + retweeted_text = Some(retweet.text.clone()); + } + } + } + + let tweet_text = tweet.text.clone(); + + let tweet_details = TweetDetails { + has_mention, + is_retweet, + retweeted_text, + user_id, + username, + display_name, + profile_image_url, + tweet_text, + }; + + info!("\n\nTweet Details: {:?}", tweet_details); + + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct TweetDetails { + tweet_text: String, + + has_mention: bool, + is_retweet: bool, + retweeted_text: Option, + + user_id : Option, + username : Option, + display_name : Option, + profile_image_url : Option, } diff --git a/src/main.rs b/src/main.rs index 9c6d3e93ab..dbd60b8e92 100644 --- a/src/main.rs +++ b/src/main.rs @@ -98,9 +98,9 @@ async fn main() -> anyhow::Result<()> in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, lang: Some("en"), place: None, possibly_sensitive: None, quoted_status_id: None, quoted_status: None, retweet_count: 0, retweeted: Some(false), - retweeted_status: Some(Tweet { coordinates: None, created_at: 2021-04-25T02:49:40Z, current_user_retweet: None, display_text_range: Some((0, 140)), + retweeted_status: Some(Tweet { coordinates: None, created_at: 2021-04-25T02:49:40Z, + current_user_retweet: None, display_text_range: Some((0, 140)), entities: TweetEntities { hashtags: [], symbols: [], urls: [UrlEntity { display_url: "twitter.com/i/web/status/1…", - expanded_url: Some("https://twitter.com/i/web/status/1386150388873715712"), range: (117, 140), url: "https://t.co/x5JPIBKr3x" }], user_mentions: [], media: None }, extended_entities: None, favorite_count: 17, favorited: Some(false), filter_level: Some(Low), id: 1386150388873715712, From 021213205a6bc65d9876ce7d0c6c424dae14ccd5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 02:54:27 -0400 Subject: [PATCH 10/19] cleanup --- src/main.rs | 114 ---------------------------------------------------- 1 file changed, 114 deletions(-) diff --git a/src/main.rs b/src/main.rs index dbd60b8e92..df067ec7e6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,6 @@ extern crate serde_derive; mod clients; mod secrets; -//mod util; use futures::TryStreamExt; use egg_mode::stream::StreamMessage; @@ -55,119 +54,6 @@ async fn main() -> anyhow::Result<()> info!("Streaming..."); - /* - - Here's an @vocodes mention by @vocodesbot: - ========================================== - - Tweet: Tweet { coordinates: None, created_at: 2021-05-09T19:00:01Z, current_user_retweet: None, - display_text_range: None, entities: TweetEntities { hashtags: [], symbols: [], urls: [], - - user_mentions: [MentionEntity { id: 1297106371238932481, range: (17, 25), name: "vocodes", screen_name: "vocodes" }], - - media: None }, extended_entities: None, favorite_count: 0, favorited: Some(false), filter_level: Some(Low), - id: 1391468014755880962, in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, - lang: Some("en"), place: None, possibly_sensitive: None, quoted_status_id: None, quoted_status: None, - retweet_count: 0, retweeted: Some(false), retweeted_status: None, - source: Some(TweetSource { name: "Twitter Web App", url: "https://mobile.twitter.com" }), - text: "Okay, so I think @vocodes mentions should work", truncated: false, - user: Some(TwitterUser { contributors_enabled: false, created_at: 2020-09-27T02:04:35Z, default_profile: true, - default_profile_image: false, description: Some("This is a bot account for @vocodes / - https://t.co/EDUojadeKa, created by @echelon. You can make it quote you by mentioning it. Please don\'t use this for evil. DEEP FAKE."), - entities: UserEntities { description: UserEntityDetail { urls: [] }, url: None }, favourites_count: 2, follow_request_sent: None, - followers_count: 30, friends_count: 1, geo_enabled: true, id: 1310037441441599488, is_translator: false, - lang: None, listed_count: 1, location: Some("Atlanta, GA"), name: "vocodes bot", profile_background_color: "F5F8FA", - profile_background_image_url: Some(""), profile_background_image_url_https: Some(""), profile_background_tile: Some(false), - profile_banner_url: None, profile_image_url: "http://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", - profile_image_url_https: "https://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", profile_link_color: "1DA1F2", - profile_sidebar_border_color: "C0DEED", profile_sidebar_fill_color: "DDEEF6", profile_text_color: "333333", - profile_use_background_image: true, protected: false, screen_name: "VocodesBot", show_all_inline_media: None, status: None, - statuses_count: 64, time_zone: None, url: Some("https://vo.codes"), utc_offset: None, verified: false, - withheld_in_countries: Some([]), withheld_scope: None }), withheld_copyright: false, withheld_in_countries: None, withheld_scope: None } - - - Here's an @vocodes retweet (not quote tweet) by @vocodesbot - ============================================================ - - Tweet: Tweet { coordinates: None, created_at: 2021-05-09T19:12:44Z, current_user_retweet: None, display_text_range: None, - entities: TweetEntities { hashtags: [], symbols: [], urls: [], - - user_mentions: [MentionEntity { id: 1297106371238932481, range: (3, 11), name: "vocodes", screen_name: "vocodes" }], - - media: None }, extended_entities: None, favorite_count: 0, favorited: Some(false), filter_level: Some(Low), id: 1391471213713805315, - in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, lang: Some("en"), place: None, - possibly_sensitive: None, quoted_status_id: None, quoted_status: None, retweet_count: 0, retweeted: Some(false), - - retweeted_status: Some(Tweet { coordinates: None, created_at: 2021-04-25T02:49:40Z, - current_user_retweet: None, display_text_range: Some((0, 140)), - entities: TweetEntities { hashtags: [], symbols: [], urls: [UrlEntity { display_url: "twitter.com/i/web/status/1…", - expanded_url: Some("https://twitter.com/i/web/status/1386150388873715712"), range: (117, 140), - url: "https://t.co/x5JPIBKr3x" }], user_mentions: [], media: None }, extended_entities: None, - favorite_count: 17, favorited: Some(false), filter_level: Some(Low), id: 1386150388873715712, - in_reply_to_user_id: None, in_reply_to_screen_name: None, in_reply_to_status_id: None, lang: Some("en"), - place: None, possibly_sensitive: Some(false), quoted_status_id: None, quoted_status: None, retweet_count: 11, - retweeted: Some(false), retweeted_status: None, source: Some(TweetSource { name: "Twitter Web App", - url: "https://mobile.twitter.com" }), - text: "You\'ll also get Joker (and JFK) in tomorrow\'s stream. I haven\'t scheduled it yet, but I\'m - thinking 2 or 3 PM EDT. I\'ll post in a bit. https://t.co/T97CR9BXcf", - truncated: true, user: Some(TwitterUser { contributors_enabled: false, created_at: 2020-08-22T09:41:05Z, - default_profile: true, default_profile_image: false, description: Some("It looks like a toy now, but - I want to democratize Hollywood with tech. \ntext to speech: https://t.co/htL5ArOIPW\ninteractive - twitch stream: https://t.co/qMvlOF5Zji\nMade by @echelon"), entities: UserEntities { description: - UserEntityDetail { urls: [] }, url: None }, favourites_count: 602, follow_request_sent: None, - followers_count: 2264, friends_count: 129, geo_enabled: false, id: 1297106371238932481, is_translator: false, - lang: None, listed_count: 7, location: Some("Atlanta, GA"), name: "vocodes", profile_background_color: "F5F8FA", - profile_background_image_url: Some(""), profile_background_image_url_https: Some(""), - profile_background_tile: Some(false), profile_banner_url: Some("https://pbs.twimg.com/profile_banners/1297106371238932481/1599623560"), - profile_image_url: "http://pbs.twimg.com/profile_images/1305039188023222273/NmVznkUO_normal.png", - profile_image_url_https: "https://pbs.twimg.com/profile_images/1305039188023222273/NmVznkUO_normal.png", - profile_link_color: "1DA1F2", profile_sidebar_border_color: "C0DEED", profile_sidebar_fill_color: "DDEEF6", - profile_text_color: "333333", profile_use_background_image: true, protected: false, screen_name: "vocodes", - show_all_inline_media: None, status: None, statuses_count: 303, time_zone: None, url: Some("https://vo.codes"), - utc_offset: None, verified: false, withheld_in_countries: Some([]), withheld_scope: None }), - withheld_copyright: false, withheld_in_countries: None, withheld_scope: None }), - source: Some(TweetSource { name: "Twitter Web App", url: "https://mobile.twitter.com" }), - text: "RT @vocodes: You\'ll also get Joker (and JFK) in tomorrow\'s stream. I haven\'t scheduled it yet, but - I\'m thinking 2 or 3 PM EDT. I\'ll post in…", truncated: false, user: Some(TwitterUser { contributors_enabled: false, - created_at: 2020-09-27T02:04:35Z, default_profile: true, default_profile_image: false, description: Some("This is a bot - account for @vocodes / https://t.co/EDUojadeKa, created by @echelon. You can make it quote you by mentioning it. - Please don\'t use this for evil. DEEP FAKE."), entities: UserEntities { description: - UserEntityDetail { urls: [] }, url: None }, favourites_count: 2, follow_request_sent: None, followers_count: 30, - friends_count: 1, geo_enabled: true, id: 1310037441441599488, is_translator: false, lang: None, listed_count: 1, - location: Some("Atlanta, GA"), name: "vocodes bot", profile_background_color: "F5F8FA", profile_background_image_url: Some(""), - profile_background_image_url_https: Some(""), profile_background_tile: Some(false), profile_banner_url: None, - profile_image_url: "http://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", - profile_image_url_https: "https://pbs.twimg.com/profile_images/1310044084506112000/vVJwVAcf_normal.png", - profile_link_color: "1DA1F2", profile_sidebar_border_color: "C0DEED", profile_sidebar_fill_color: "DDEEF6", - profile_text_color: "333333", profile_use_background_image: true, protected: false, screen_name: "VocodesBot", - show_all_inline_media: None, status: None, statuses_count: 65, time_zone: None, url: Some("https://vo.codes"), utc_offset: None, - verified: false, withheld_in_countries: Some([]), withheld_scope: None }), withheld_copyright: false, withheld_in_countries: None, - withheld_scope: None } - - - */ - - /* - let stream = egg_mode::stream::filter() - .follow(&[1297106371238932481]) // vocodes - .track(&["vocodes"]) - //.language(&["en"]) - .start(&twitter_access_token) - .try_for_each(|m| { - if let StreamMessage::Tweet(tweet) = m { - info!("Tweet: {:?}", &tweet); - } else { - info!("Other: {:?}", &m); - } - futures::future::ok(()) - }); - - if let Err(e) = stream.await { - warn!("Disconnected. Stream error: {:?}", e); - } - - Ok(())*/ - let twitter_client = TwitterClient::new(twitter_access_token); twitter_client.main_loop().await; From 134ff704bfb9ebf239d04f076842206c454448be Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 03:13:19 -0400 Subject: [PATCH 11/19] Add protos submodule --- .gitmodules | 3 +++ README.md | 7 +++++++ protos/storyteller-protos | 1 + 3 files changed, 11 insertions(+) create mode 100644 .gitmodules create mode 160000 protos/storyteller-protos diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..e2aaee7783 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "protos/storyteller-protos"] + path = protos/storyteller-protos + url = git@github.com:storytold/storyteller-protos.git diff --git a/README.md b/README.md index cd7a86141d..5dfc73fbe2 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,10 @@ twitter-firehose Ingest events from Twitter and send them to Redis PubSub +Protos +------ +Protos are defined in a git submodule: + +``` +git submodule add git@github.com:storytold/storyteller-protos.git +``` diff --git a/protos/storyteller-protos b/protos/storyteller-protos new file mode 160000 index 0000000000..c61a069a6f --- /dev/null +++ b/protos/storyteller-protos @@ -0,0 +1 @@ +Subproject commit c61a069a6ffcd6fdc15684e3086c48373a670fd0 From ebd98a3571bb56484b10165ed237475d29ec9bd5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 11:31:31 -0400 Subject: [PATCH 12/19] fixed type issues --- build.rs | 17 +++ protos/storyteller-protos | 2 +- src/clients/twitter_client.rs | 85 ++++++++--- src/main.rs | 6 +- src/proto_codegen/storyteller.pubsub.rs | 192 ++++++++++++++++++++++++ src/protos.rs | 63 ++++++++ 6 files changed, 339 insertions(+), 26 deletions(-) create mode 100644 build.rs create mode 100644 src/proto_codegen/storyteller.pubsub.rs create mode 100644 src/protos.rs diff --git a/build.rs b/build.rs new file mode 100644 index 0000000000..2b28014080 --- /dev/null +++ b/build.rs @@ -0,0 +1,17 @@ +extern crate anyhow; // TODO: No! +extern crate prost_build; + +use std::env; + +fn main() -> anyhow::Result<()> { + // NB: Control prost output dir with env var + // Typically these land in `target/`, but I want to source control these. + env::set_var("OUT_DIR", "src/proto_codegen"); + + prost_build::compile_protos( + &[ + "protos/storyteller-protos/protos/pubsub-events/pubsub-events.proto" + ], + &["protos/"])?; + Ok(()) +} diff --git a/protos/storyteller-protos b/protos/storyteller-protos index c61a069a6f..67b750d956 160000 --- a/protos/storyteller-protos +++ b/protos/storyteller-protos @@ -1 +1 @@ -Subproject commit c61a069a6ffcd6fdc15684e3086c48373a670fd0 +Subproject commit 67b750d956c9e62051e67acab850a3cbc4398ad9 diff --git a/src/clients/twitter_client.rs b/src/clients/twitter_client.rs index 6645e20b9b..18796da830 100644 --- a/src/clients/twitter_client.rs +++ b/src/clients/twitter_client.rs @@ -1,22 +1,43 @@ use crate::AnyhowResult; -use log::{info, debug, warn}; -use futures::TryStreamExt; +use anyhow::anyhow; use egg_mode::stream::StreamMessage; -use std::time::Duration; -use std::thread; use egg_mode::tweet::Tweet; +use futures::TryStreamExt; +use log::{info, debug, warn}; +use std::thread; +use std::time::Duration; +use crate::clients::redis_client::RedisClient; +use futures::executor::block_on; const VOCODES_USER_ID : u64 = 1297106371238932481; +/// The client logic pub struct TwitterClient { access_token: egg_mode::Token, + redis_client: RedisClient, +} + +/// The parts of tweets we care about +#[derive(Clone, Debug)] +pub struct TweetDetails { + pub tweet_text: String, + + pub has_mention: bool, + pub is_retweet: bool, + pub retweeted_text: Option, + + pub user_id : Option, + pub username : Option, + pub display_name : Option, + pub profile_image_url : Option, } impl TwitterClient { - pub fn new(access_token: egg_mode::Token) -> Self { + pub fn new(access_token: egg_mode::Token, redis_client: RedisClient) -> Self { Self { access_token, + redis_client, } } @@ -37,19 +58,43 @@ impl TwitterClient { .track(&["vocodes"]) .start(&self.access_token) .try_for_each(|m| { + // NB: Odd implementation is due to future type juggling. + // I should fix this, but I'm too lazy and have more work to do. if let StreamMessage::Tweet(tweet) = m { - self.handle_tweet(tweet); + self.handle_tweet_mapping_errors(Some(tweet)) } else { info!("Other (non-Tweet) message: {:?}", &m); + self.handle_tweet_mapping_errors(None) } - futures::future::ok(()) }); - Ok(stream.await?) + match stream.await { + Ok(_) => Ok(()), + Err(_) => Err(anyhow!("Stream error")), // NB: loss of error due to shenanigans + } } - fn handle_tweet(&self, tweet: Tweet) -> AnyhowResult<()> { - info!("Tweet: {:?}", &tweet); + // NB: This is some nonsense to satisfy type requirements. + // I'm too lazy to figure out why try_for_each demands egg_mode's Error. + async fn handle_tweet_mapping_errors(&self, tweet: Option) + -> Result<(), egg_mode::error::Error> + { + // NB: This is horrible. + self.handle_tweet(tweet) + .await + .map_err(|e| { + warn!("Error during stream: {:?}", e); + egg_mode::error::Error::BadUrl + }) + } + + async fn handle_tweet(&self, tweet: Option) -> AnyhowResult<()> { + let tweet = match tweet { + None => return Ok(()), + Some(tweet) => tweet, + }; + + debug!("Tweet: {:?}", &tweet); let mut user_id : Option = None; let mut username : Option = None; @@ -101,22 +146,14 @@ impl TwitterClient { tweet_text, }; - info!("\n\nTweet Details: {:?}", tweet_details); + info!("Tweet Details: {:?}", tweet_details); + self.publish_tweet(tweet_details).await?; Ok(()) } + + async fn publish_tweet(&self, tweet_details: TweetDetails) -> AnyhowResult<()> { + Ok(()) + } } -#[derive(Clone, Debug)] -pub struct TweetDetails { - tweet_text: String, - - has_mention: bool, - is_retweet: bool, - retweeted_text: Option, - - user_id : Option, - username : Option, - display_name : Option, - profile_image_url : Option, -} diff --git a/src/main.rs b/src/main.rs index df067ec7e6..3f74b98d25 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ extern crate serde_derive; mod clients; +mod protos; mod secrets; use futures::TryStreamExt; @@ -54,7 +55,10 @@ async fn main() -> anyhow::Result<()> info!("Streaming..."); - let twitter_client = TwitterClient::new(twitter_access_token); + let twitter_client = TwitterClient::new( + twitter_access_token, + redis_client, + ); twitter_client.main_loop().await; Ok(()) diff --git a/src/proto_codegen/storyteller.pubsub.rs b/src/proto_codegen/storyteller.pubsub.rs new file mode 100644 index 0000000000..65fbfa61f2 --- /dev/null +++ b/src/proto_codegen/storyteller.pubsub.rs @@ -0,0 +1,192 @@ +/// Ingestion platforms are "dumb" and shouldn't have much logic in them. +/// (Granted, we might eventually grow to send them back messages.) +/// +/// The "enricher" is the smarts. It does denylisting, rate limiting, +/// service calls, transforms, etc. +/// +/// What if we need to scale? Several ideas: Sharding (redis db indices), +/// brokers with read-once semantics (our own intelligence - UUIDs with +/// transactional semantics in MySQL). +/// +/// Events coming from multiple sources +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PubsubEventPayloadV1 { + /// Source of the event. + #[prost(enumeration="pubsub_event_payload_v1::IngestionSourceType", optional, tag="1")] + pub ingestion_source_type: ::core::option::Option, + #[prost(enumeration="pubsub_event_payload_v1::IngestionPayloadType", optional, tag="2")] + pub ingestion_payload_type: ::core::option::Option, + /// A binary-encoded proto whose type is based on IngestionSourceType + #[prost(bytes="vec", optional, tag="3")] + pub ingestion_source_data: ::core::option::Option<::prost::alloc::vec::Vec>, + /// A binary-encoded proto whose type is based on IngestionPayloadType + #[prost(bytes="vec", optional, tag="4")] + pub ingestion_payload_data: ::core::option::Option<::prost::alloc::vec::Vec>, + /// An optional debug message. + #[prost(string, optional, tag="5")] + pub debug_message: ::core::option::Option<::prost::alloc::string::String>, +} +/// Nested message and enum types in `PubsubEventPayloadV1`. +pub mod pubsub_event_payload_v1 { + /// TODO: This should all be merged in the future, but I need to settle on proto2 vs proto3 + /// And I also need to figure out package names, etc. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum IngestionSourceType { + IstDoNotUse = 0, + IstAdminPanel = 1, + IstVocodesWebsite = 2, + IstTwitch = 3, + IstTwitter = 4, + IstDiscord = 5, + IstYoutube = 6, + IstReddit = 7, + IstTiktok = 8, + } + /// These will differ from the "enriched" types. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] + #[repr(i32)] + pub enum IngestionPayloadType { + IptDoNotUse = 0, + /// Twitch-specific events. + /// + /// This can encode lots of things. + TwitchMessage = 100, + TwitchSubscribed = 101, + /// Twitter-specific events. + TwitterFollow = 200, + TwitterRetweet = 201, + TwitterMention = 202, + /// Discord-specific events. + DiscordMessage = 300, + DiscordJoined = 301, + /// YouTube-specific events. + YoutubeSubscribed = 400, + /// Reddit-specific events. + RedditMessage = 500, + } +} +// ============================== SOURCE PROTOS ============================== // + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitchMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(int64, optional, tag="1")] + pub user_id: ::core::option::Option, + #[prost(string, optional, tag="2")] + pub username: ::core::option::Option<::prost::alloc::string::String>, + #[prost(bool, optional, tag="3")] + pub user_is_mod: ::core::option::Option, + #[prost(bool, optional, tag="4")] + pub user_is_subscribed: ::core::option::Option, + /// Optional. Name of the channel, prefixed with hash. eg. `#vocodes`. + #[prost(string, optional, tag="5")] + pub channel: ::core::option::Option<::prost::alloc::string::String>, +} +/// Optional. Fields describing the user that sent the message. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterMetadata { + // TODO: This is a u64. We need to be careful with math to assure non-overflow. + //optional int64 user_id = 1; + + #[prost(string, optional, tag="2")] + pub username: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag="3")] + pub display_name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag="4")] + pub avatar_url: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionDiscordMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionYoutubeMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionRedditMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTikTokMetadata { + /// Optional. Fields describing the user that sent the message. + #[prost(string, optional, tag="1")] + pub username: ::core::option::Option<::prost::alloc::string::String>, +} +// ============================== TWITCH PAYLOAD PROTOS ============================== // + +/// These are raw messages from Twitch that we send to the PubSub enricher. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitchMessage { + /// TODO: DEPRECATE THESE + #[prost(int64, optional, tag="1")] + pub user_id: ::core::option::Option, + #[prost(string, optional, tag="2")] + pub username: ::core::option::Option<::prost::alloc::string::String>, + #[prost(bool, optional, tag="3")] + pub is_mod: ::core::option::Option, + #[prost(bool, optional, tag="4")] + pub is_subscribed: ::core::option::Option, + /// TODO: DEPRECATE THESE + /// Name of the channel, prefixed with hash. eg. `#vocodes`. + #[prost(string, optional, tag="5")] + pub channel: ::core::option::Option<::prost::alloc::string::String>, + /// Actual message. Doesn't need to be trimmed. + #[prost(string, optional, tag="6")] + pub message_contents: ::core::option::Option<::prost::alloc::string::String>, +} +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitchSubscribed { +} +// ============================== TWITTER PAYLOAD PROTOS ============================== // + +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterFollow { +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterRetweet { + /// Intentionally blank for now + #[prost(string, optional, tag="1")] + pub original_text: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionTwitterMention { + /// Intentionally blank for now + #[prost(string, optional, tag="1")] + pub text: ::core::option::Option<::prost::alloc::string::String>, +} +// ============================== DISCORD PAYLOAD PROTOS ============================== // + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionDiscordMessage { + /// Raw message. Needs to be trimmed + #[prost(string, optional, tag="6")] + pub message_contents: ::core::option::Option<::prost::alloc::string::String>, +} +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionDiscordJoined { +} +// ============================== YOUTUBE PAYLOAD PROTOS ============================== // + +/// Intentionally blank for now +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionYoutubeSubscribed { +} +// ============================== REDDIT PAYLOAD PROTOS ============================== // + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IngestionRedditMessage { + /// Raw message. Needs to be trimmed + #[prost(string, optional, tag="6")] + pub message_contents: ::core::option::Option<::prost::alloc::string::String>, +} diff --git a/src/protos.rs b/src/protos.rs new file mode 100644 index 0000000000..96cc94b27e --- /dev/null +++ b/src/protos.rs @@ -0,0 +1,63 @@ +use anyhow::anyhow; +use crate::AnyhowResult; +use log::{info, warn, debug}; +use crate::clients::twitter_client::TweetDetails; + +// Include the protos +pub mod protos { + include!("proto_codegen/storyteller.pubsub.rs"); +} + +// Binary encode a proto. +pub fn binary_encode_proto(proto: impl prost::Message) -> AnyhowResult> { + let mut buffer : Vec = Vec::with_capacity(proto.encoded_len()); + let encode_result = proto.encode(&mut buffer); + + match encode_result { + Err(e) => { + Err(anyhow!("Inner proto encode result: {:?}", e)) + } + Ok(_) => { + Ok(buffer) + } + } +} + +pub fn mention_to_proto<'a>(tweet_details: TweetDetails) -> AnyhowResult +{ + let mut payload_proto = protos::PubsubEventPayloadV1::default(); + + payload_proto.ingestion_source_type = + Some(protos::pubsub_event_payload_v1::IngestionSourceType::IstTwitter as i32); + + let binary_twitter_metadata = { + let mut twitter_metadata = protos::IngestionTwitterMetadata::default(); + // TODO: Don't want to overflow. Represent as bytes in the meantime + // twitter_metadata.user_id = Some(tweet_details.user_id as i64); + twitter_metadata.username = tweet_details.username.clone(); + twitter_metadata.display_name = tweet_details.display_name.clone(); + twitter_metadata.avatar_url = tweet_details.profile_image_url.clone(); + + debug!("Twitter Metadata Proto: {:?}", twitter_metadata); + + binary_encode_proto(twitter_metadata) + }?; + + payload_proto.ingestion_source_data = Some(binary_twitter_metadata); + + payload_proto.ingestion_payload_type = + Some(protos::pubsub_event_payload_v1::IngestionPayloadType::TwitterMention as i32); + + let binary_twitter_mention = { + let mut twitter_mention = protos::IngestionTwitterMention::default(); + twitter_mention.text = Some(tweet_details.tweet_text.clone()); + + debug!("Twitter Mention Proto: {:?}", twitter_mention); + + binary_encode_proto(twitter_mention) + }?; + + payload_proto.ingestion_payload_data = Some(binary_twitter_mention); + + Ok(payload_proto) +} From 8bd0f22ff73eac58594c63062649299b55bdbd35 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 18:51:57 -0400 Subject: [PATCH 13/19] this builds, but isn't correct --- src/clients/twitter_client.rs | 67 +++++++++++++++++++++++++++++------ src/main.rs | 5 +-- src/protos.rs | 39 ++++++++++++++++++++ 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/src/clients/twitter_client.rs b/src/clients/twitter_client.rs index 18796da830..b091be2bb3 100644 --- a/src/clients/twitter_client.rs +++ b/src/clients/twitter_client.rs @@ -8,13 +8,16 @@ use std::thread; use std::time::Duration; use crate::clients::redis_client::RedisClient; use futures::executor::block_on; +use crate::protos::{mention_to_proto, binary_encode_proto}; +use std::sync::{RwLock, Arc}; const VOCODES_USER_ID : u64 = 1297106371238932481; /// The client logic pub struct TwitterClient { access_token: egg_mode::Token, - redis_client: RedisClient, + redis_client: RedisClientWrapper, + redis_publish_topic: String, } /// The parts of tweets we care about @@ -32,16 +35,41 @@ pub struct TweetDetails { pub profile_image_url : Option, } +// NB: We're using interior mutability. +struct RedisClientWrapper { + redis_client: Arc::>, +} + +impl RedisClientWrapper { + pub fn new(redis_client: RedisClient) -> Self { + Self { + redis_client: Arc::new(RwLock::new(redis_client)) + } + } + pub async fn publish_bytes(&self, channel: &str, message: &[u8]) -> AnyhowResult { + let mut redis_client = match self.redis_client.write() { + Ok(rc) => rc, + Err(e) => return Err(anyhow!("Error: {:?}", )), + }; + redis_client.publish_bytes(channel, message).await + } +} + impl TwitterClient { - pub fn new(access_token: egg_mode::Token, redis_client: RedisClient) -> Self { + pub fn new( + access_token: egg_mode::Token, + redis_client: RedisClient, + redis_publish_topic: String) -> Self + { Self { access_token, - redis_client, + redis_client: RedisClientWrapper::new(redis_client), + redis_publish_topic, } } - pub async fn main_loop(&self) { + pub async fn main_loop(&mut self) { loop { info!("Connecting to twitter stream..."); if let Err(e) = self.run_stream().await { @@ -52,20 +80,23 @@ impl TwitterClient { } } - pub async fn run_stream(&self) -> AnyhowResult<()> { + pub async fn run_stream(&mut self) -> AnyhowResult<()> { let stream = egg_mode::stream::filter() .follow(&[VOCODES_USER_ID]) // vocodes .track(&["vocodes"]) .start(&self.access_token) + //.fold(&self) .try_for_each(|m| { // NB: Odd implementation is due to future type juggling. // I should fix this, but I'm too lazy and have more work to do. if let StreamMessage::Tweet(tweet) = m { - self.handle_tweet_mapping_errors(Some(tweet)) + self.handle_tweet_mapping_errors(Some(tweet)); } else { info!("Other (non-Tweet) message: {:?}", &m); - self.handle_tweet_mapping_errors(None) + self.handle_tweet_mapping_errors(None); } + + futures::future::ok(()) }); match stream.await { @@ -76,7 +107,7 @@ impl TwitterClient { // NB: This is some nonsense to satisfy type requirements. // I'm too lazy to figure out why try_for_each demands egg_mode's Error. - async fn handle_tweet_mapping_errors(&self, tweet: Option) + async fn handle_tweet_mapping_errors(&mut self, tweet: Option) -> Result<(), egg_mode::error::Error> { // NB: This is horrible. @@ -88,7 +119,7 @@ impl TwitterClient { }) } - async fn handle_tweet(&self, tweet: Option) -> AnyhowResult<()> { + async fn handle_tweet(&mut self, tweet: Option) -> AnyhowResult<()> { let tweet = match tweet { None => return Ok(()), Some(tweet) => tweet, @@ -152,7 +183,23 @@ impl TwitterClient { Ok(()) } - async fn publish_tweet(&self, tweet_details: TweetDetails) -> AnyhowResult<()> { + async fn publish_tweet(&mut self, tweet_details: TweetDetails) -> AnyhowResult<()> { + let binary_proto = if !tweet_details.is_retweet && tweet_details.has_mention { + // NB: Retweets can have mentions of the target user. + let proto = mention_to_proto(tweet_details)?; + binary_encode_proto(proto)? + } else if tweet_details.is_retweet { + let proto = mention_to_proto(tweet_details)?; + binary_encode_proto(proto)? + } else { + return Ok(()); + }; + + self.redis_client.publish_bytes( + &self.redis_publish_topic, + &binary_proto + ).await?; + Ok(()) } } diff --git a/src/main.rs b/src/main.rs index 3f74b98d25..b2c5685d69 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,7 +45,7 @@ async fn main() -> anyhow::Result<()> let mut redis_client = RedisClient::new( &secrets.redis, - redis_max_retry_count + redis_max_retry_count, ); redis_client.connect().await?; @@ -55,9 +55,10 @@ async fn main() -> anyhow::Result<()> info!("Streaming..."); - let twitter_client = TwitterClient::new( + let mut twitter_client = TwitterClient::new( twitter_access_token, redis_client, + redis_publish_topic, ); twitter_client.main_loop().await; diff --git a/src/protos.rs b/src/protos.rs index 96cc94b27e..4d41449cb4 100644 --- a/src/protos.rs +++ b/src/protos.rs @@ -61,3 +61,42 @@ pub fn mention_to_proto<'a>(tweet_details: TweetDetails) -> AnyhowResult(tweet_details: TweetDetails) -> AnyhowResult +{ + let mut payload_proto = protos::PubsubEventPayloadV1::default(); + + payload_proto.ingestion_source_type = + Some(protos::pubsub_event_payload_v1::IngestionSourceType::IstTwitter as i32); + + let binary_twitter_metadata = { + let mut twitter_metadata = protos::IngestionTwitterMetadata::default(); + // TODO: Don't want to overflow. Represent as bytes in the meantime + // twitter_metadata.user_id = Some(tweet_details.user_id as i64); + twitter_metadata.username = tweet_details.username.clone(); + twitter_metadata.display_name = tweet_details.display_name.clone(); + twitter_metadata.avatar_url = tweet_details.profile_image_url.clone(); + + debug!("Twitter Metadata Proto: {:?}", twitter_metadata); + + binary_encode_proto(twitter_metadata) + }?; + + payload_proto.ingestion_source_data = Some(binary_twitter_metadata); + + payload_proto.ingestion_payload_type = + Some(protos::pubsub_event_payload_v1::IngestionPayloadType::TwitterMention as i32); + + let binary_twitter_retweet = { + let mut twitter_retweet = protos::IngestionTwitterRetweet::default(); + twitter_retweet.original_text = tweet_details.retweeted_text.clone(); + + debug!("Twitter Retweet Proto: {:?}", twitter_retweet); + + binary_encode_proto(twitter_retweet) + }?; + + payload_proto.ingestion_payload_data = Some(binary_twitter_retweet); + + Ok(payload_proto) +} From 4e2c18bece9b4a4b2c91c91a4ed9ead56e756050 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 18:56:52 -0400 Subject: [PATCH 14/19] this builds; unsure if correct --- src/clients/twitter_client.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/clients/twitter_client.rs b/src/clients/twitter_client.rs index b091be2bb3..4e62c34589 100644 --- a/src/clients/twitter_client.rs +++ b/src/clients/twitter_client.rs @@ -86,17 +86,21 @@ impl TwitterClient { .track(&["vocodes"]) .start(&self.access_token) //.fold(&self) - .try_for_each(|m| { + .try_for_each(|m: StreamMessage| { // NB: Odd implementation is due to future type juggling. // I should fix this, but I'm too lazy and have more work to do. - if let StreamMessage::Tweet(tweet) = m { - self.handle_tweet_mapping_errors(Some(tweet)); + let future = if let StreamMessage::Tweet(tweet) = m { + self.handle_tweet_mapping_errors(Some(tweet)) } else { info!("Other (non-Tweet) message: {:?}", &m); - self.handle_tweet_mapping_errors(None); + self.handle_tweet_mapping_errors(None) + }; + + match block_on(future) { + Ok(x) => futures::future::ok(x), + Err(e) => futures::future::err(e), } - futures::future::ok(()) }); match stream.await { From 91f01342830eccb7e0ed0df54c0464bff483dcdd Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 22:24:53 -0400 Subject: [PATCH 15/19] rename app --- Cargo.lock | 2 +- Cargo.toml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 299706b42b..a55205f19e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1767,7 +1767,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] -name = "twitter-firehose" +name = "twitter-ingestion-client" version = "0.0.0" dependencies = [ "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 6d980e7e7b..88f303588a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "twitter-firehose" +name = "twitter-ingestion-client" version = "0.0.0" edition = "2018" authors = [ @@ -8,7 +8,7 @@ authors = [ ] [[bin]] -name = "twitter-firehose" +name = "twitter-ingestion-client" path = "src/main.rs" [[bin]] @@ -35,3 +35,4 @@ egg-mode = "0.15.0" [build-dependencies] anyhow = "1.0.38" prost-build = "0.7.0" + From 7a3e98bd6b72479ce6c3f5ffdc5c69f21d492c3b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 23:29:17 -0400 Subject: [PATCH 16/19] add dockerfile / build config --- .github/workflows/docker-publish.yml | 56 ++++++++++++++++++++++++++++ Dockerfile | 54 +++++++++++++++++++++++++++ README.md | 4 +- 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 Dockerfile diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000..b2fe622601 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,56 @@ +# Adapted from +# https://github.com/actions/starter-workflows/ci/docker-publish.yml +name: Publish Docker Container to GHCR +on: + push: + # Publish `$default-branch` as Docker `latest` image. + branches: + #- $default-branch + - master +env: + IMAGE_NAME: twitter-ingestion-client + +jobs: + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME + + - name: Log into GitHub Container Registry + # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` + run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "$default-branch" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + + SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12) + + docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA + docker push $IMAGE_ID:$SHORT_SHA + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..ce8c5d3cfe --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +# Unsure if I want to create minimal images yet that run under Alpine Linux. +# I have instructions for that here: https://hub.docker.com/r/echelon/testcontainer/dockerfile +FROM ubuntu:xenial as build +WORKDIR /tmp + +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + curl \ + libssl-dev \ + libssl1.0.0 \ + pkg-config \ + unzip \ + wget +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- --default-toolchain stable -y + +COPY Cargo.lock . +COPY Cargo.toml . +COPY src/ ./src +COPY protos/ ./protos + +RUN $HOME/.cargo/bin/cargo fetch + +#RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build +RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build --release + +# Final image +FROM ubuntu:xenial +WORKDIR / + +# NB: Comment this out for non-debug images +# TODO: Figure out how this is done elsewhere with just the static binaries +RUN apt-get update \ + && apt-get install -y \ + curl \ + wget + +#COPY --from=build /tmp/target/debug/twitter-ingestion-client / +COPY --from=build /tmp/target/release/twitter-ingestion-client / + +# SSL certs are required for crypto +COPY --from=build /etc/ssl /etc/ssl + +# Required dynamically linked libraries +COPY --from=build /lib/x86_64-linux-gnu/libssl.* /lib/x86_64-linux-gnu/ +COPY --from=build /lib/x86_64-linux-gnu/libcrypto.* /lib/x86_64-linux-gnu/ + +# Make sure all the links resolve +RUN ldd twitter-ingestion-client + +EXPOSE 8080 +CMD LD_LIBRARY_PATH=/usr/lib /twitter-ingestion-client + diff --git a/README.md b/README.md index 5dfc73fbe2..f62a0f583b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -twitter-firehose -================ +twitter-ingestion-client +======================== Ingest events from Twitter and send them to Redis PubSub From 3ca18afceb90f57503fb997766cabe7ca3633926 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 10 May 2021 23:49:06 -0400 Subject: [PATCH 17/19] GITHUB_TOKEN instead of dangerous PAT --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b2fe622601..3adb5227bd 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -25,7 +25,7 @@ jobs: - name: Log into GitHub Container Registry # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` - run: echo "${{ secrets.CR_PAT }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin - name: Push image to GitHub Container Registry run: | From c14f8bddd5a6e3b0e51a9020cdee52b71990e1e8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 12 May 2021 03:17:28 -0400 Subject: [PATCH 18/19] new dockerfile, GHA, minor updates --- .github/workflows/docker-publish.yml | 21 +++++++++++++++------ Cargo.lock | 11 +++++++++-- Cargo.toml | 2 +- Dockerfile | 27 +++++++++++---------------- src/main.rs | 2 +- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3adb5227bd..0f93ec142b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,10 +1,12 @@ # Adapted from # https://github.com/actions/starter-workflows/ci/docker-publish.yml +# If using base images, don't forget to add READ access, eg. from a package settings page like: +# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access name: Publish Docker Container to GHCR on: push: - # Publish `$default-branch` as Docker `latest` image. branches: + # NB: Default-branch doesn't work, despite Github's documentation. #- $default-branch - master env: @@ -16,16 +18,23 @@ jobs: push: runs-on: ubuntu-latest if: github.event_name == 'push' + permissions: + packages: write + contents: read steps: - uses: actions/checkout@v2 - - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + #username: ${{ github.repository_owner }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Log into GitHub Container Registry - # TODO: Create a PAT with `read:packages` and `write:packages` scopes and save it as an Actions secret `CR_PAT` - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" - name: Push image to GitHub Container Registry run: | diff --git a/Cargo.lock b/Cargo.lock index a55205f19e..7c8b31c916 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,6 +378,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dtoa" version = "0.4.8" @@ -386,10 +392,11 @@ checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "easyenv" -version = "0.0.4" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516fc6eb3e59cee34859122f05f0c94e28266c76b2ed0efe8d6611ed9c2082c1" +checksum = "852054bee794f364fbe890bd947cbf678c35226b81ede17720d6a17fc687a0d4" dependencies = [ + "dotenv", "env_logger", "log", ] diff --git a/Cargo.toml b/Cargo.toml index 88f303588a..7e64d38498 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ path = "src/main_get_access_token.rs" [dependencies] anyhow = "1.0.38" -easyenv = "0.0.4" +easyenv = "0.0.5" futures = "0.3.14" log = "0.4.14" prost = "0.7.0" diff --git a/Dockerfile b/Dockerfile index ce8c5d3cfe..e9ea566816 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,9 @@ -# Unsure if I want to create minimal images yet that run under Alpine Linux. -# I have instructions for that here: https://hub.docker.com/r/echelon/testcontainer/dockerfile -FROM ubuntu:xenial as build +# Custom base image +# Make sure to add this repository so it has read acces to the base image: +# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access +FROM ghcr.io/storytold/docker-base-images-rust-ssl:latest as build WORKDIR /tmp -RUN apt-get update \ - && apt-get install -y \ - build-essential \ - curl \ - libssl-dev \ - libssl1.0.0 \ - pkg-config \ - unzip \ - wget -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- --default-toolchain stable -y - COPY Cargo.lock . COPY Cargo.toml . COPY src/ ./src @@ -23,12 +12,18 @@ COPY protos/ ./protos RUN $HOME/.cargo/bin/cargo fetch #RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build -RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build --release +RUN LD_LIBRARY_PATH=/usr/lib:${LD_LIBRARY_PATH} $HOME/.cargo/bin/cargo build --release --bin twitter-ingestion-client # Final image FROM ubuntu:xenial WORKDIR / +# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md +LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com' +LABEL org.opencontainers.image.url='https://github.com/storytold/twitter-ingestion-client' +LABEL org.opencontainers.image.documentation='https://github.com/storytold/twitter-ingestion-client' +LABEL org.opencontainers.image.source='https://github.com/storytold/twitter-ingestion-client' + # NB: Comment this out for non-debug images # TODO: Figure out how this is done elsewhere with just the static binaries RUN apt-get update \ diff --git a/src/main.rs b/src/main.rs index b2c5685d69..6e767d42e3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,7 +32,7 @@ const ENV_REDIS_MAX_RETRY_COUNT_DEFAULT : u32 = 3; #[tokio::main] async fn main() -> anyhow::Result<()> { - easyenv::init_env_logger(None); + easyenv::init_all_with_default_logging(None); let redis_publish_topic = easyenv::get_env_string_or_default( ENV_PUBLISH_TOPIC, ENV_PUBLISH_TOPIC_DEFAULT); From 033c6026ad381d358a0a11929fa7913096c96fae Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 12 May 2021 03:31:40 -0400 Subject: [PATCH 19/19] switch to env-based config --- .env-template | 16 ++++++++++++++++ .gitignore | 1 + src/main.rs | 51 +++++++++++++++++++++++++++++++++++++++++--------- src/secrets.rs | 39 +++++++++++++++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 .env-template diff --git a/.env-template b/.env-template new file mode 100644 index 0000000000..aa6b89c535 --- /dev/null +++ b/.env-template @@ -0,0 +1,16 @@ +# Redis connection +REDIS_USERNAME= +REDIS_PASSWORD= +REDIS_HOST= +REDIS_PORT= +REDIS_USES_TLS= + +# Redis publishing +#REDIS_TOPIC=firehose +#REDIS_MAX_RETRY_COUNT=3 + +# Twitter secrets +TWITTER_API_KEY= +TWITTER_API_SECRET= +TWITTER_ACCESS_KEY= +TWITTER_ACCESS_SECRET= diff --git a/.gitignore b/.gitignore index a9a3e92446..aee3b3b46c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *~ +.env .idea/ secrets.toml secrets.yaml diff --git a/src/main.rs b/src/main.rs index 6e767d42e3..b54272ec4f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,12 +11,14 @@ mod clients; mod protos; mod secrets; -use futures::TryStreamExt; -use egg_mode::stream::StreamMessage; -use crate::secrets::Secrets; use crate::clients::redis_client::RedisClient; -use log::{info, warn, debug}; use crate::clients::twitter_client::TwitterClient; +use crate::secrets::RedisSecrets; +use crate::secrets::Secrets; +use crate::secrets::TwitterSecrets; +use egg_mode::stream::StreamMessage; +use futures::TryStreamExt; +use log::{info, warn, debug}; pub type AnyhowResult = anyhow::Result; @@ -34,6 +36,8 @@ async fn main() -> anyhow::Result<()> { easyenv::init_all_with_default_logging(None); + info!("Reading env config..."); + let redis_publish_topic = easyenv::get_env_string_or_default( ENV_PUBLISH_TOPIC, ENV_PUBLISH_TOPIC_DEFAULT); @@ -41,19 +45,24 @@ async fn main() -> anyhow::Result<()> ENV_REDIS_MAX_RETRY_COUNT, ENV_REDIS_MAX_RETRY_COUNT_DEFAULT)?; - let secrets = Secrets::from_file("secrets.toml")?; + //let secrets = Secrets::from_file("secrets.toml")?; + + let redis_secrets = get_redis_secrets()?; + let twitter_secrets = get_twitter_secrets()?; let mut redis_client = RedisClient::new( - &secrets.redis, + &redis_secrets, redis_max_retry_count, ); + info!("Connecting to Redis..."); + redis_client.connect().await?; info!("Verifying Twitter access token..."); - let twitter_access_token = secrets.twitter.verify_access_token().await?; + let twitter_access_token = twitter_secrets.verify_access_token().await?; - info!("Streaming..."); + info!("Streaming from Twitter..."); let mut twitter_client = TwitterClient::new( twitter_access_token, @@ -63,4 +72,28 @@ async fn main() -> anyhow::Result<()> twitter_client.main_loop().await; Ok(()) -} \ No newline at end of file +} + +fn get_redis_secrets() -> AnyhowResult { + Ok(RedisSecrets::new( + &easyenv::get_env_string_or_default( + "REDIS_USERNAME", ""), + &easyenv::get_env_string_or_default( + "REDIS_PASSWORD", ""), + &easyenv::get_env_string_or_default( + "REDIS_HOST", ""), + easyenv::get_env_num::( + "REDIS_PORT", 6379)?, + easyenv::get_env_bool_or_default( + "REDIS_USES_TLS", false), + )) +} + +fn get_twitter_secrets() -> AnyhowResult { + Ok(TwitterSecrets::new( + &easyenv::get_env_string_required("TWITTER_API_KEY")?, + &easyenv::get_env_string_required("TWITTER_API_SECRET")?, + &easyenv::get_env_string_optional("TWITTER_ACCESS_KEY"), + &easyenv::get_env_string_optional("TWITTER_ACCESS_SECRET"), + )) +} diff --git a/src/secrets.rs b/src/secrets.rs index 3170a031cd..5c3f7dcd32 100644 --- a/src/secrets.rs +++ b/src/secrets.rs @@ -15,6 +15,7 @@ pub struct RedisSecrets { pub password: String, pub host: String, pub port: u32, + pub uses_tls: bool, } #[derive(Deserialize, Clone)] @@ -38,12 +39,48 @@ impl Secrets { } impl RedisSecrets { + pub fn new(username: &str, password: &str, host: &str, port: u32, uses_tls: bool) -> Self { + Self { + username: username.to_string(), + password: password.to_string(), + host: host.to_string(), + port, + uses_tls, + } + } + pub fn connection_url(&self) -> String { - format!("rediss://{}:{}@{}:{}", self.username, self.password, self.host, self.port) + let protocol = if self.uses_tls { "rediss" } else { "redis" }; + let mut auth = "".to_string(); + + if !self.username.is_empty() { + if !self.password.is_empty() { + auth = format!("{}:{}@", self.username, self.password); + } else { + auth = format!("{}@", self.username); + } + } else if !self.password.is_empty() { + auth = format!("default:{}@", self.password); + } + + format!("{}://{}{}:{}", protocol, auth, self.host, self.port) } } impl TwitterSecrets { + pub fn new(api_key: &str, + api_secret_key: &str, + access_key: &Option, + access_secret: &Option) -> Self { + + Self { + api_key: api_key.to_string(), + api_secret_key: api_secret_key.to_string(), + access_key: access_key.clone(), + access_secret: access_secret.clone(), + } + } + /// This assumes we have a valid pair of access (key, secret). pub async fn verify_access_token(&self) -> AnyhowResult { let consumer_keypair = self.make_consumer_keypair();